Node节点部署

一、部署kubelect

二进制包准备 将软件包从linux-node1复制到linux-node2、linux-node3中去

[root@linux-node1 ~]# cd /usr/local/src/kubernetes/server/bin/
[root@linux-node1 bin]# cp kubelet kube-proxy /opt/kubernetes/bin/
[root@linux-node1 bin]# scp kubelet kube-proxy 192.168.56.12:/opt/kubernetes/bin/
[root@linux-node1 bin]# scp kubelet kube-proxy 192.168.56.13:/opt/kubernetes/bin/

创建角色绑定

[root@linux-node1 bin]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap
clusterrolebinding.rbac.authorization.k8s.io "kubelet-bootstrap" created

创建 kubelet bootstrapping kubeconfig 文件 设置集群参数

[root@linux-node1 bin]# cd /usr/local/src/ssl/
[root@linux-node1 ssl]# pwd
/usr/local/src/ssl
[root@linux-node1 ssl]#  kubectl config set-cluster kubernetes \
    --certificate-authority=/opt/kubernetes/ssl/ca.pem \
    --embed-certs=true \
    --server=https://192.168.56.11:6443 \
    --kubeconfig=bootstrap.kubeconfig
Cluster "kubernetes" set.

设置客户端认证参数

[root@linux-node1 ssl]#  kubectl config set-credentials kubelet-bootstrap \
    --token=ad6d5bb607a186796d8861557df0d17f \
    --kubeconfig=bootstrap.kubeconfig
User "kubelet-bootstrap" set.

设置上下文参数

[root@linux-node1 ssl]# kubectl config set-context default \
    --cluster=kubernetes \
    --user=kubelet-bootstrap \
    --kubeconfig=bootstrap.kubeconfig
Context "default" created.

选择默认上下文

[root@linux-node1 ssl]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
Switched to context "default".
[root@linux-node1 ssl]# ll bootstrap.kubeconfig
-rw------- 1 root root 2167 May 31 15:14 bootstrap.kubeconfig

[root@linux-node1 ssl]# cp bootstrap.kubeconfig /opt/kubernetes/cfg
#后续如果添加节点，需要把此证书拷贝至节点上
[root@linux-node1 ssl]# scp bootstrap.kubeconfig 192.168.56.12:/opt/kubernetes/cfg
[root@linux-node1 ssl]# scp bootstrap.kubeconfig 192.168.56.13:/opt/kubernetes/cfg

二、部署kubelet 1.设置CNI支持

#提示所有node节点都要执行此操作
####################linux-node1节点为master节点###############################
[root@linux-node1 ssl]# mkdir -p /etc/cni/net.d
[root@linux-node1 ssl]# vim /etc/cni/net.d/10-default.conf
{
        "name": "flannel",
        "type": "flannel",
        "delegate": {
            "bridge": "docker0",
            "isDefaultGateway": true,
            "mtu": 1400
        }
}
######################linux-node2节点#############################
[root@linux-node2 ~]# mkdir -p /etc/cni/net.d
[root@linux-node2 ~]# vim /etc/cni/net.d/10-default.conf
{
        "name": "flannel",
        "type": "flannel",
        "delegate": {
            "bridge": "docker0",
            "isDefaultGateway": true,
            "mtu": 1400
        }
}
######################linux-node3节点#############################
[root@linux-node3 ~]# mkdir -p /etc/cni/net.d
[root@linux-node3 ~]# vim /etc/cni/net.d/10-default.conf
{
        "name": "flannel",
        "type": "flannel",
        "delegate": {
            "bridge": "docker0",
            "isDefaultGateway": true,
            "mtu": 1400
        }
}

创建kubelet目录

[root@linux-node1 ssl]# mkdir /var/lib/kubelet
[root@linux-node2 ~]# mkdir /var/lib/kubelet
[root@linux-node3 ~]# mkdir /var/lib/kubelet

创建kubelet服务配置

[root@linux-node1 ssl]# vim /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \
  --address=192.168.56.11 \
  --hostname-override=192.168.56.11 \
  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
  --experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
  --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
  --cert-dir=/opt/kubernetes/ssl \
  --network-plugin=cni \
  --cni-conf-dir=/etc/cni/net.d \
  --cni-bin-dir=/opt/kubernetes/bin/cni \
  --cluster-dns=10.1.0.2 \
  --cluster-domain=cluster.local. \
  --hairpin-mode hairpin-veth \
  --allow-privileged=true \
  --fail-swap-on=false \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log
Restart=on-failure
RestartSec=5

同步配置

[root@linux-node1 ssl]# scp /usr/lib/systemd/system/kubelet.service 192.168.56.12:/usr/lib/systemd/system/kubelet.service
kubelet.service                                                             
[root@linux-node1 ssl]# scp /usr/lib/systemd/system/kubelet.service 192.168.56.13:/usr/lib/systemd/system/kubelet.service
kubelet.service            

[root@linux-node2 ~]# vim /usr/lib/systemd/system/kubelet.service
[root@linux-node2 ~]# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \
  --address=192.168.56.12 \
  --hostname-override=192.168.56.12 \
  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
  --experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
  --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
  --cert-dir=/opt/kubernetes/ssl \
  --network-plugin=cni \
  --cni-conf-dir=/etc/cni/net.d \
  --cni-bin-dir=/opt/kubernetes/bin/cni \
  --cluster-dns=10.1.0.2 \
  --cluster-domain=cluster.local. \
  --hairpin-mode hairpin-veth \
  --allow-privileged=true \
  --fail-swap-on=false \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log
Restart=on-failure
RestartSec=5

[root@linux-node3 ~]# vim /usr/lib/systemd/system/kubelet.service
[root@linux-node3 ~]# cat /usr/lib/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/opt/kubernetes/bin/kubelet \
  --address=192.168.56.13 \
  --hostname-override=192.168.56.13 \
  --pod-infra-container-image=mirrorgooglecontainers/pause-amd64:3.0 \
  --experimental-bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \
  --kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \
  --cert-dir=/opt/kubernetes/ssl \
  --network-plugin=cni \
  --cni-conf-dir=/etc/cni/net.d \
  --cni-bin-dir=/opt/kubernetes/bin/cni \
  --cluster-dns=10.1.0.2 \
  --cluster-domain=cluster.local. \
  --hairpin-mode hairpin-veth \
  --allow-privileged=true \
  --fail-swap-on=false \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log
Restart=on-failure
RestartSec=5

启动服务

#linux-node1节点可以不启动，此节点为master节点
[root@linux-node1 ssl]# systemctl daemon-reload

#######################linux-node2启动Kubelet###############
[root@linux-node2 ~]# systemctl daemon-reload
[root@linux-node2 ~]# systemctl enable kubelet
[root@linux-node2 ~]# systemctl start kubelet



#######################linux-node3启动Kubelet##############
[root@linux-node3 ~]# systemctl daemon-reload
[root@linux-node3 ~]# systemctl enable kubelet
[root@linux-node3 ~]# systemctl start kubelet

查看服务状态

systemctl status kubelet

查看csr请求 注意是在linux-node1上执行

[root@linux-node1 ssl]# kubectl get csr
NAME                                                   AGE       REQUESTOR           CONDITION
node-csr-4WjZeYEZtU9TO8vFm_KncCEetdiZpXpBxfL-wJ9GRyk   21s       kubelet-bootstrap   Pending
node-csr-zxEKag7ig1-X8RBd_wfuLoblPv7WnwLSNTj-ozXvjGI   24s       kubelet-bootstrap   Pending

[root@linux-node2 ~]# cd /opt/kubernetes/ssl/
[root@linux-node2 ssl]# ll
total 44
-rw-r--r-- 1 root root  290 May 30 21:10 ca-config.json
-rw-r--r-- 1 root root 1001 May 30 21:10 ca.csr
-rw------- 1 root root 1675 May 30 21:10 ca-key.pem
-rw-r--r-- 1 root root 1359 May 30 21:10 ca.pem
-rw------- 1 root root 1675 May 30 22:32 etcd-key.pem
-rw-r--r-- 1 root root 1436 May 30 22:32 etcd.pem
-rw------- 1 root root  227 May 31 15:30 kubelet-client.key
-rw-r--r-- 1 root root 2185 May 31 15:30 kubelet.crt
-rw------- 1 root root 1679 May 31 15:30 kubelet.key
-rw------- 1 root root 1675 May 31 14:34 kubernetes-key.pem
-rw-r--r-- 1 root root 1610 May 31 14:34 kubernetes.pem

[root@linux-node3 ~]# cd /opt/kubernetes/ssl/
[root@linux-node3 ssl]# ll
total 44
-rw-r--r-- 1 root root  290 May 30 21:11 ca-config.json
-rw-r--r-- 1 root root 1001 May 30 21:11 ca.csr
-rw------- 1 root root 1675 May 30 21:11 ca-key.pem
-rw-r--r-- 1 root root 1359 May 30 21:11 ca.pem
-rw------- 1 root root 1675 May 30 22:32 etcd-key.pem
-rw-r--r-- 1 root root 1436 May 30 22:32 etcd.pem
-rw------- 1 root root  227 May 31 15:30 kubelet-client.key
-rw-r--r-- 1 root root 2185 May 31 15:30 kubelet.crt
-rw------- 1 root root 1675 May 31 15:30 kubelet.key
-rw------- 1 root root 1675 May 31 14:34 kubernetes-key.pem
-rw-r--r-- 1 root root 1610 May 31 14:34 kubernetes.pem

批准kubelet的TLS证书请求

[root@linux-node1 ssl]# kubectl get csr|grep 'Pending' | awk 'NR>0{print $1}'| xargs kubectl certificate approve
certificatesigningrequest.certificates.k8s.io "node-csr-4WjZeYEZtU9TO8vFm_KncCEetdiZpXpBxfL-wJ9GRyk" approved
certificatesigningrequest.certificates.k8s.io "node-csr-zxEKag7ig1-X8RBd_wfuLoblPv7WnwLSNTj-ozXvjGI" approved
[root@linux-node1 ssl]# kubectl get csr
NAME                                                   AGE       REQUESTOR           CONDITION
node-csr-4WjZeYEZtU9TO8vFm_KncCEetdiZpXpBxfL-wJ9GRyk   3m        kubelet-bootstrap   Approved,Issued
node-csr-zxEKag7ig1-X8RBd_wfuLoblPv7WnwLSNTj-ozXvjGI   3m        kubelet-bootstrap   Approved,Issued

执行完毕后，查看节点状态已经是Ready的状态了

[root@linux-node1 ssl]# kubectl get node
NAME            STATUS    ROLES     AGE       VERSION
192.168.56.12   Ready     <none>    3m        v1.10.1
192.168.56.13   Ready     <none>    3m        v1.10.1

[root@linux-node2 ssl]# ll kubelet*
-rw-r--r-- 1 root root 1046 May 31 15:33 kubelet-client.crt
-rw------- 1 root root  227 May 31 15:30 kubelet-client.key
-rw-r--r-- 1 root root 2185 May 31 15:30 kubelet.crt
-rw------- 1 root root 1679 May 31 15:30 kubelet.key

[root@linux-node3 ssl]# ll kubelet*
-rw-r--r-- 1 root root 1046 May 31 15:33 kubelet-client.crt
-rw------- 1 root root  227 May 31 15:30 kubelet-client.key
-rw-r--r-- 1 root root 2185 May 31 15:30 kubelet.crt
-rw------- 1 root root 1675 May 31 15:30 kubelet.key

三、部署kubernetes Proxy

配置kube-proxy使用LVS

[root@linux-node1 ssl]# yum install -y ipvsadm ipset conntrack    
[root@linux-node2 ssl]# yum install -y ipvsadm ipset conntrack
[root@linux-node3 ssl]# yum install -y ipvsadm ipset conntrack

创建kube-proxy证书请求

[root@linux-node1 ssl]# vim kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

生成证书

[root@linux-node1 ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \
    -ca-key=/opt/kubernetes/ssl/ca-key.pem \
    -config=/opt/kubernetes/ssl/ca-config.json \
    -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy
2018/05/31 15:47:07 [INFO] generate received request
2018/05/31 15:47:07 [INFO] received CSR
2018/05/31 15:47:07 [INFO] generating key: rsa-2048
2018/05/31 15:47:08 [INFO] encoded CSR
2018/05/31 15:47:08 [INFO] signed certificate with serial number 686335705927831639688007943181627318620328401639
2018/05/31 15:47:08 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

分发证书至所有Node节点

[root@linux-node1 ssl]# cp kube-proxy*.pem /opt/kubernetes/ssl/
[root@linux-node1 ssl]# scp kube-proxy*.pem 192.168.56.12:/opt/kubernetes/ssl/
[root@linux-node1 ssl]# scp kube-proxy*.pem 192.168.56.13:/opt/kubernetes/ssl/

创建kube-proxy配置文件

[root@linux-node1 ssl]# kubectl config set-cluster kubernetes \
    --certificate-authority=/opt/kubernetes/ssl/ca.pem \
    --embed-certs=true \
    --server=https://192.168.56.11:6443 \
    --kubeconfig=kube-proxy.kubeconfig
Cluster "kubernetes" set.
[root@linux-node1 ssl]# ll kube-proxy*
-rw-r--r-- 1 root root 1009 May 31 15:47 kube-proxy.csr
-rw-r--r-- 1 root root  230 May 31 15:46 kube-proxy-csr.json
-rw------- 1 root root 1679 May 31 15:47 kube-proxy-key.pem
-rw------- 1 root root 2011 May 31 15:48 kube-proxy.kubeconfig
-rw-r--r-- 1 root root 1403 May 31 15:47 kube-proxy.pem
[root@linux-node1 ssl]# kubectl config set-credentials kube-proxy \
    --client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
    --client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
    --embed-certs=true \
    --kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.

[root@linux-node1 ssl]# kubectl config set-context default \
    --cluster=kubernetes \
    --user=kube-proxy \
    --kubeconfig=kube-proxy.kubeconfig
Context "default" created.

[root@linux-node1 ssl]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
Switched to context "default".

分发kubeconfig配置文件

[root@linux-node1 ssl]# cp kube-proxy.kubeconfig /opt/kubernetes/cfg/
[root@linux-node1 ssl]# scp kube-proxy.kubeconfig 192.168.56.12:/opt/kubernetes/cfg/
[root@linux-node1 ssl]# scp kube-proxy.kubeconfig 192.168.56.13:/opt/kubernetes/cfg/

创建kube-proxy服务配置

[root@linux-node1 ssl]# mkdir /var/lib/kube-proxy
[root@linux-node2 ssl]# mkdir /var/lib/kube-proxy
[root@linux-node3 ssl]# mkdir /var/lib/kube-proxy
[root@linux-node1 ssl]# vim /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \
  --bind-address=192.168.56.11 \
  --hostname-override=192.168.56.11 \
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig \
--masquerade-all \
  --feature-gates=SupportIPVSProxyMode=true \
  --proxy-mode=ipvs \
  --ipvs-min-sync-period=5s \
  --ipvs-sync-period=5s \
  --ipvs-scheduler=rr \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log

Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target


[root@linux-node1 ssl]# scp /usr/lib/systemd/system/kube-proxy.service 192.168.56.12:/usr/lib/systemd/system/kube-proxy.service
[root@linux-node1 ssl]# scp /usr/lib/systemd/system/kube-proxy.service 192.168.56.13:/usr/lib/systemd/system/kube-proxy.service

[root@linux-node2 ssl]# vim /usr/lib/systemd/system/kube-proxy.service
[root@linux-node2 ssl]# cat /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \
  --bind-address=192.168.56.12 \
  --hostname-override=192.168.56.12 \
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig \
--masquerade-all \
  --feature-gates=SupportIPVSProxyMode=true \
  --proxy-mode=ipvs \
  --ipvs-min-sync-period=5s \
  --ipvs-sync-period=5s \
  --ipvs-scheduler=rr \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log

Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

[root@linux-node3 ssl]# cat /usr/lib/systemd/system/kube-proxy.service
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/opt/kubernetes/bin/kube-proxy \
  --bind-address=192.168.56.13 \
  --hostname-override=192.168.56.13 \
  --kubeconfig=/opt/kubernetes/cfg/kube-proxy.kubeconfig \
--masquerade-all \
  --feature-gates=SupportIPVSProxyMode=true \
  --proxy-mode=ipvs \
  --ipvs-min-sync-period=5s \
  --ipvs-sync-period=5s \
  --ipvs-scheduler=rr \
  --logtostderr=true \
  --v=2 \
  --logtostderr=false \
  --log-dir=/opt/kubernetes/log

Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

所有node节点启动服务

#####################linux-node2启动服务##############################
[root@linux-node2 ssl]# systemctl daemon-reload
[root@linux-node2 ssl]# systemctl enable kube-proxy
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.
[root@linux-node2 ssl]# systemctl start kube-proxy
[root@linux-node2 ssl]# systemctl status kube-proxy

#####################linux-node3启动服务##############################
[root@linux-node3 ssl]# systemctl daemon-reload
[root@linux-node3 ssl]# systemctl enable kube-proxy
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-proxy.service to /usr/lib/systemd/system/kube-proxy.service.
[root@linux-node3 ssl]# systemctl start kube-proxy

查看LVS状态

[root@linux-node2 ssl]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.1.0.1:443 rr persistent 10800
  -> 192.168.56.11:6443           Masq    1      0          0


[root@linux-node3 ssl]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.1.0.1:443 rr persistent 10800
  -> 192.168.56.11:6443           Masq    1      0          0

如果你在两台实验机器都安装了kubelet和proxy服务，使用下面的命令可以检查状态：
[root@linux-node1 ssl]#  kubectl get node
NAME            STATUS    ROLES     AGE       VERSION
192.168.56.12   Ready     <none>    22m       v1.10.1
192.168.56.13   Ready     <none>    3m        v1.10.1

 

[root@linux-node1 ssl]# kubectl get service
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.1.0.1     <none>        443/TCP   1h