五.二进制部署kubernetes node篇（kube-proxy，kubelet）

1. 将kubelet-bootstrap用户绑定到系统集群角色

[root@k8s-master kubernetes]# kubectl create clusterrolebinding kubelet-bootstrap \
> --clusterrole=system:node-bootstrapper \
> --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created

2. 创建kubeconfig文件

[root@k8s-master cert]# cat kubeconfig.sh 
# 创建 TLS Bootstrapping Token
#BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')
BOOTSTRAP_TOKEN=4644a663112ab3bcb0c5f91ce5b92b8f # 这个token 是安装master节点时候生成的 在/opt/kubernetes/cfg/token.csv

cat > token.csv <<EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

#----------------------

APISERVER=$1
SSL_DIR=$2

# 创建kubelet bootstrapping kubeconfig 
export KUBE_APISERVER="https://$APISERVER:6443"

# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=$SSL_DIR/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=bootstrap.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=${BOOTSTRAP_TOKEN} \
  --kubeconfig=bootstrap.kubeconfig

# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig

# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

#----------------------

# 创建kube-proxy kubeconfig文件

kubectl config set-cluster kubernetes \
  --certificate-authority=$SSL_DIR/ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \
  --client-certificate=$SSL_DIR/kube-proxy.pem \
  --client-key=$SSL_DIR/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

[root@k8s-master cert]# sh kubeconfig.sh  192.168.1.119 ./    # 这里需要指定之前安装kube-apiserver生成的ssl证书目录 
[root@k8s-master cert]# ls  #方便对比
admin.csr       admin-key.pem  bootstrap.kubeconfig  ca.csr       ca-key.pem  k8s-cert.sh    kube-proxy.csr       kube-proxy-key.pem     kube-proxy.pem  server-csr.json  server.pem
admin-csr.json  admin.pem      ca-config.json        ca-csr.json  ca.pem      kubeconfig.sh  kube-proxy-csr.json  kube-proxy.kubeconfig  server.csr      server-key.pem   token.csv

　3.将文件拷贝到2个node节点上

[root@k8s-master cert]# scp bootstrap.kubeconfig  kube-proxy.kubeconfig  root@192.168.1.120:/opt/kubernetes/cfg/
[root@k8s-master cert]# scp bootstrap.kubeconfig  kube-proxy.kubeconfig  root@192.168.1.121:/opt/kubernetes/cfg/

　4.将安装包内的 kube-proxy kubelet 拷贝到2个node节点上

[root@k8s-master bin]# scp kube-proxy kubelet  root@192.168.1.120:/opt/kubernetes/bin/
[root@k8s-master bin]# scp kube-proxy kubelet  root@192.168.1.121:/opt/kubernetes/bin/

　5.配置kubelet，kube-proxy

   1> kubelet 脚本信息如下

[root@k8s-node1 ~]# cat kubelet.sh 
#!/bin/bash

NODE_ADDRESS=$1
DNS_SERVER_IP=${2:-"10.0.0.2"}

cat <<EOF >/opt/kubernetes/cfg/kubelet

KUBELET_OPTS="--logtostderr=true \\
--v=4 \\
--address=${NODE_ADDRESS} \\
--hostname-override=${NODE_ADDRESS} \\
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\
--experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\
--config=/opt/kubernetes/cfg/kubelet.config \\
--cert-dir=/opt/kubernetes/ssl \\
--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0"

EOF

cat <<EOF >/opt/kubernetes/cfg/kubelet.config

kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: ${NODE_ADDRESS}
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
- ${DNS_SERVER_IP} 
clusterDomain: cluster.local.
failSwapOn: false
authentication:
  anonymous:
    enabled: true

EOF

cat <<EOF >/usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet

　2>kube-proxy脚本信息如下　

[root@k8s-node1 ~]# cat proxy.sh 
#!/bin/bash

NODE_ADDRESS=$1

cat <<EOF >/opt/kubernetes/cfg/kube-proxy

KUBE_PROXY_OPTS="--logtostderr=true \\
--v=4 \\
--hostname-override=${NODE_ADDRESS} \\
--cluster-cidr=10.0.0.0/24 \\
--proxy-mode=ipvs \\
--kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig"

EOF

cat <<EOF >/usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy

[root@k8s-node1 ~]# sh kubelet.sh 192.168.1.120
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
[root@k8s-node1 ~]# sh proxy.sh 192.168.1.120  
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.

# master节点执行 授权
[root@k8s-master ~]# kubectl  get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-0bxwJb8lMJ8x0Kfi5gzXquKYDUKMncZfZpie0SDXUak   10m   kubelet-bootstrap   Pending
[root@k8s-master ~]# kubectl  certificate approve node-csr-0bxwJb8lMJ8x0Kfi5gzXquKYDUKMncZfZpie0SDXUak 
certificatesigningrequest.certificates.k8s.io/node-csr-0bxwJb8lMJ8x0Kfi5gzXquKYDUKMncZfZpie0SDXUak approved
[root@k8s-master ~]# kubectl  get nodes
NAME            STATUS   ROLES    AGE   VERSION
192.168.1.120   Ready    <none>   33s   v1.12.1


[root@k8s-node2 ~]# sh kubelet.sh 192.168.1.121
Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service.
[root@k8s-node2 ~]# sh proxy.sh 192.168.1.121  
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.

# master节点执行 授权
[root@k8s-master bin]# kubectl  get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-0bxwJb8lMJ8x0Kfi5gzXquKYDUKMncZfZpie0SDXUak   18m   kubelet-bootstrap   Approved,Issued
node-csr-UX-xX5ftoR8Kd1utTDFAlnMfP0_jVjKU-_JEiJ81vU0   12s   kubelet-bootstrap   Pending
[root@k8s-master bin]# kubectl  certificate approve node-csr-UX-xX5ftoR8Kd1utTDFAlnMfP0_jVjKU-_JEiJ81vU0
certificatesigningrequest.certificates.k8s.io/node-csr-UX-xX5ftoR8Kd1utTDFAlnMfP0_jVjKU-_JEiJ81vU0 approved
[root@k8s-master bin]# kubectl  get nodes
NAME            STATUS   ROLES    AGE     VERSION
192.168.1.120   Ready    <none>   8m40s   v1.12.1
192.168.1.121   Ready    <none>   29s     v1.12.1