install_k8s1.13.1_node.sh

1.masterIP为k8s master的IP,nodeIP为k8s node的IP,CLUSTER_CIDR为etcd设置的pod网段

2.建立k8s node与k8s master的ssh 信任关系，让k8s node 可以免密ssh k8s master

3.运行如下脚本，添加1.13.1的k8s node

export masterIP=10.45.52.249

export nodeIP=10.81.64.173

export CLUSTER_CIDR="172.22.0.0/16"

yum install -y conntrack ipvsadm ipset jq sysstat curl iptables libseccomp

systemctl stop firewalld

systemctl disable firewalld

systemctl stop iptables

systemctl disable iptables

swapoff -a

sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

setenforce 0

modprobe br_netfilter

modprobe ip_vs

mkdir -p /app/k8s

cd /app/k8s

cat > kubernetes.conf <<EOF

net.ipv4.ip_forward=1

vm.swappiness=0

vm.overcommit_memory=1

vm.panic_on_oom=0

fs.inotify.max_user_watches=89100

fs.file-max=52706963

fs.nr_open=52706963

net.ipv6.conf.all.disable_ipv6=1

net.netfilter.nf_conntrack_max=2310720

kernel.pid_max=1000000

net.ipv4.tcp_syncookies = 1

net.ipv4.tcp_tw_reuse = 1

net.ipv4.tcp_fin_timeout = 30

net.ipv4.conf.default.rp_filter = 0

net.ipv4.conf.all.rp_filter = 0

net.ipv4.conf.eth0.rp_filter = 0

net.ipv4.tcp_timestamps=0

net.ipv4.tcp_tw_recycle=0

net.ipv4.tcp_max_tw_buckets = 3000

EOF

systemctl stop kubelet

systemctl stop kube-proxy

systemctl stop flanneld

systemctl stop docker

systemctl disable kubelet

systemctl disable kube-proxy

systemctl disable flanneld

systemctl disable docker

  

rm -rf /etc/sysctl.d/kubernetes.conf

cp kubernetes.conf /etc/sysctl.d/kubernetes.conf

sysctl -p /etc/sysctl.d/kubernetes.conf

mount -t cgroup -o cpu,cpuacct none /sys/fs/cgroup/cpu,cpuacct

rm -rf /etc/kubernetes

 

mkdir -p /etc/kubernetes/cert

scp root@${masterIP}:/etc/kubernetes/cert/ca*.pem /etc/kubernetes/cert/

scp root@${masterIP}:/etc/kubernetes/cert/ca-config.json /etc/kubernetes/cert

scp root@${masterIP}:/usr/local/bin/kubectl /usr/local/bin/

rm -rf ~/.kube

mkdir -p ~/.kube

scp root@${masterIP}:~/.kube/config ~/.kube/

scp root@${masterIP}:/usr/local/bin/{flanneld,mk-docker-opts.sh} /usr/local/bin/

rm -rf  /etc/flanneld

mkdir -p /etc/flanneld/cert

scp root@${masterIP}:/etc/flanneld/cert/flanneld*.pem /etc/flanneld/cert

export IFACE=eth0

export ETCD_ENDPOINTS="https://${masterIP}:2379"

export FLANNEL_ETCD_PREFIX="/kubernetes/network"

cat > flanneld.service << EOF

[Unit]

Description=Flanneld overlay address etcd agent

After=network.target

After=network-online.target

Wants=network-online.target

After=etcd.service

Before=docker.service

[Service]

Type=notify

ExecStart=/usr/local/bin/flanneld \\

-etcd-cafile=/etc/kubernetes/cert/ca.pem \\

-etcd-certfile=/etc/flanneld/cert/flanneld.pem \\

-etcd-keyfile=/etc/flanneld/cert/flanneld-key.pem \\

-etcd-endpoints=${ETCD_ENDPOINTS} \\

-etcd-prefix=${FLANNEL_ETCD_PREFIX} \\

-iface=${IFACE}

ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker

Restart=on-failure

[Install]

WantedBy=multi-user.target

RequiredBy=docker.service

EOF

rm -rf /etc/systemd/system/flanneld.service

cp flanneld.service /etc/systemd/system/

systemctl daemon-reload

systemctl enable flanneld

systemctl restart flanneld

 

scp root@${masterIP}:/usr/local/bin/docker* /usr/local/bin/

scp root@${masterIP}:/etc/systemd/system/docker.service /etc/systemd/system/

rm -rf  /etc/docker

mkdir -p /etc/docker/

scp root@${masterIP}:/etc/docker/daemon.json /etc/docker/

systemctl daemon-reload

systemctl enable docker

systemctl restart docker

 

export KUBE_APISERVER="https://${masterIP}:6443"

export node_name="kube-node2"

scp root@${masterIP}:/usr/local/bin/kube* /usr/local/bin/

# 创建 token,注意下面的node_name必须是字符+数字+下划线,而且每个节点node_name必须唯一，不能重复，比如kube-node1，kube-node2，如果写的是IP(${nodeIP})就无法生成token

export BOOTSTRAP_TOKEN=$(kubeadm token create \

--description kubelet-bootstrap-token \

--groups system:bootstrappers:${node_name} \

--kubeconfig ~/.kube/config)

# 设置集群参数

kubectl config set-cluster kubernetes \

--certificate-authority=/etc/kubernetes/cert/ca.pem \

--embed-certs=true \

--server=${KUBE_APISERVER} \

--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置客户端认证参数

kubectl config set-credentials kubelet-bootstrap \

--token=${BOOTSTRAP_TOKEN} \

--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置上下文参数

kubectl config set-context default \

--cluster=kubernetes \

--user=kubelet-bootstrap \

--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置默认上下文

kubectl config use-context default --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

kubeadm token list --kubeconfig ~/.kube/config

rm -rf  /etc/kubernetes/kubelet-bootstrap.kubeconfig

cp kubelet-bootstrap-${node_name}.kubeconfig /etc/kubernetes/kubelet-bootstrap.kubeconfig

 

cat > kubelet.config.json <<EOF

{

"kind": "KubeletConfiguration",

"apiVersion": "kubelet.config.k8s.io/v1beta1",

"authentication": {

"x509": {

"clientCAFile": "/etc/kubernetes/cert/ca.pem"

},

"webhook": {

"enabled": true,

"cacheTTL": "2m0s"

},

"anonymous": {

"enabled": false

}

},

"authorization": {

"mode": "Webhook",

"webhook": {

"cacheAuthorizedTTL": "5m0s",

"cacheUnauthorizedTTL": "30s"

}

},

"address": "${nodeIP}",

"port": 10250,

"readOnlyPort": 0,

"cgroupDriver": "cgroupfs",

"hairpinMode": "promiscuous-bridge",

"serializeImagePulls": false,

"featureGates": {

"RotateKubeletClientCertificate": true,

"RotateKubeletServerCertificate": true

},

"clusterDomain": "cluster.local.",

"clusterDNS": ["10.254.0.2"]

}

EOF

rm -rf /etc/kubernetes/kubelet.config.json

cp kubelet.config.json /etc/kubernetes/

  

#下面这个node_name决定了kubectl get no显示的名字，为了方便管理，这里统一写成node节点的IP

export node_name="${nodeIP}"

mkdir -p /var/lib/kubelet

cat > kubelet.service <<EOF

[Unit]

Description=Kubernetes Kubelet

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

After=docker.service

Requires=docker.service

[Service]

WorkingDirectory=/var/lib/kubelet

ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/cpuset/system.slice/kubelet.service

ExecStartPre=/usr/bin/mkdir -p /sys/fs/cgroup/hugetlb/system.slice/kubelet.service

ExecStart=/usr/local/bin/kubelet \\

--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \\

--cert-dir=/etc/kubernetes/cert \\

--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\

--config=/etc/kubernetes/kubelet.config.json \\

--hostname-override=${node_name} \\

--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest \\

--allow-privileged=true \\

--alsologtostderr=true \\

--logtostderr=false \\

--log-dir=/app/log/kubernetes \\

--enforce-node-allocatable=pods,kube-reserved,system-reserved \\

--kube-reserved-cgroup=/system.slice/kubelet.service \\

--system-reserved-cgroup=/system.slice \\

--kube-reserved=cpu=500m,memory=1Gi,ephemeral-storage=1Gi \\

--system-reserved=cpu=500m,memory=2Gi,ephemeral-storage=5Gi \\

--eviction-hard=imagefs.available<5%,memory.available<500Mi,nodefs.available<5% \\

--v=0

Restart=on-failure

RestartSec=5

[Install]

WantedBy=multi-user.target

EOF

rm -rf /etc/systemd/system/kubelet.service

cp kubelet.service /etc/systemd/system/

mkdir -p /app/log/kubernetes

systemctl daemon-reload

systemctl enable kubelet

systemctl restart kubelet

sleep 10

kubectl get csr | awk '{print $1}' | grep -v "NAME"| xargs  kubectl certificate approve

scp root@${masterIP}:/etc/kubernetes/kube-proxy.kubeconfig /etc/kubernetes/

 

cat > kube-proxy.config.yaml <<EOF

apiVersion: kubeproxy.config.k8s.io/v1alpha1

bindAddress: ${nodeIP}

clientConnection:

 kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig

clusterCIDR: ${CLUSTER_CIDR}

healthzBindAddress: ${nodeIP}:10256

hostnameOverride: ${nodeIP}

kind: KubeProxyConfiguration

metricsBindAddress: ${nodeIP}:10249

mode: "ipvs"

EOF

rm -rf /etc/kubernetes/kube-proxy.config.yaml

cp kube-proxy.config.yaml /etc/kubernetes/

mkdir -p /var/lib/kube-proxy

cat > kube-proxy.service <<EOF

[Unit]

Description=Kubernetes Kube-Proxy Server

Documentation=https://github.com/GoogleCloudPlatform/kubernetes

After=network.target

[Service]

WorkingDirectory=/var/lib/kube-proxy

ExecStart=/usr/local/bin/kube-proxy \\

--config=/etc/kubernetes/kube-proxy.config.yaml \\

--alsologtostderr=true \\

--logtostderr=false \\

--log-dir=/app/log/kubernetes \\

--v=0

Restart=on-failure

RestartSec=5

LimitNOFILE=65536

[Install]

WantedBy=multi-user.target

EOF

rm -rf /etc/systemd/system/kube-proxy.service

cp kube-proxy.service /etc/systemd/system/

mkdir -p /var/lib/kube-proxy

mkdir -p /app/log/kubernetes

systemctl daemon-reload

systemctl enable kube-proxy

systemctl restart kube-proxy