一键部署Kubernetes高可用集群

三台master，四台node，系统版本为CentOS7

IP	ROLE
172.60.0.226	master01
172.60.0.86	master02
172.60.0.106	master03
172.60.0.227	node01
172.60.0.228	node02
172.60.0.44	node03
172.60.0.46	node04

这里安装的kubernetes版本为1.5.1，docker版本为1.12.3

三个master节点通过keepalived实现高可用。结构如下图，参考官网。

以下是脚本内容：

# vim k8s-deploy.sh
 
#!/bin/bash
set -x
set -e
 
HTTP_SERVER=172.60.0.43:8000
KUBE_HA=true
 
KUBE_REPO_PREFIX=gcr.io/google_containers
KUBE_ETCD_IMAGE=quay.io/coreos/etcd:v3.0.15
 
root=$(id -u)
if [ "$root" -ne 0 ] ;then
    echo must run as root
    exit 1
fi
 
kube::install_docker()
{
    set +e
    docker info> /dev/null 2>&1
    i=$?
    set -e
    if [ $i -ne 0 ]; then
        curl -L http://$HTTP_SERVER/rpms/docker.tar.gz > /tmp/docker.tar.gz
        tar zxf /tmp/docker.tar.gz -C /tmp
        yum localinstall -y /tmp/docker/*.rpm
        systemctl enable docker.service && systemctl start docker.service
        kube::config_docker
    fi
    echo docker has been installed
    rm -rf /tmp/docker /tmp/docker.tar.gz
}
 
kube::config_docker()
{
    setenforce 0 > /dev/null 2>&1 && sed -i -e 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
 
    sysctl -w net.bridge.bridge-nf-call-iptables=1
    sysctl -w net.bridge.bridge-nf-call-ip6tables=1
cat <<EOF >>/etc/sysctl.conf
    net.bridge.bridge-nf-call-ip6tables = 1
    net.bridge.bridge-nf-call-iptables = 1
EOF
 
    mkdir -p /etc/systemd/system/docker.service.d
cat <<EOF >/etc/systemd/system/docker.service.d/10-docker.conf
[Service]
    ExecStart=
    ExecStart=/usr/bin/dockerd -s overlay --selinux-enabled=false
EOF
 
    systemctl daemon-reload && systemctl restart docker.service
}
 
kube::load_images()
{
    mkdir -p /tmp/k8s
 
    images=(
        kube-apiserver-amd64_v1.5.1
        kube-controller-manager-amd64_v1.5.1
        kube-scheduler-amd64_v1.5.1
        kube-proxy-amd64_v1.5.1
        pause-amd64_3.0
        kube-discovery-amd64_1.0
        kubedns-amd64_1.9
        exechealthz-amd64_1.2
        kube-dnsmasq-amd64_1.4
        dnsmasq-metrics-amd64_1.0
        etcd_v3.0.15
        flannel-amd64_v0.7.0
    )
 
    for i in "${!images[@]}"; do
        ret=$(docker images | awk 'NR!=1{print $1"_"$2}'| grep $KUBE_REPO_PREFIX/${images[$i]} | wc -l)
        if [ $ret -lt 1 ];then
            curl -L http://$HTTP_SERVER/images/${images[$i]}.tar o /tmp/k8s/${images[$i]}.tar
            docker load -i /tmp/k8s/${images[$i]}.tar
        fi
    done
 
    rm /tmp/k8s* -rf
}
 
kube::install_bin()
{
    set +e
    which kubeadm > /dev/null 2>&1
    i=$?
    set -e
    if [ $i -ne 0 ]; then
        curl -L http://$HTTP_SERVER/rpms/k8s.tar.gz > /tmp/k8s.tar.gz
        tar zxf /tmp/k8s.tar.gz -C /tmp
        yum localinstall -y  /tmp/k8s/*.rpm
        rm -rf /tmp/k8s*
        systemctl enable kubelet.service && systemctl start kubelet.service && rm -rf /etc/kubernetes
    fi
}
 
kube::wait_apiserver()
{
    until curl http://127.0.0.1:8080; do sleep 1; done
}
 
kube::disable_static_pod()
{
    # remove the waring log in kubelet
    sed -i 's/--pod-manifest-path=\/etc\/kubernetes\/manifests//g' /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
    systemctl daemon-reload && systemctl restart kubelet.service
}
 
kube::get_env()
{
  HA_STATE=$1
  [ $HA_STATE == "MASTER" ] && HA_PRIORITY=200 || HA_PRIORITY=`expr 200 - ${RANDOM} / 1000 + 1`
  KUBE_VIP=$(echo $2 |awk -F= '{print $2}')
  VIP_PREFIX=$(echo ${KUBE_VIP} | cut -d . -f 1,2,3)
  #dhcp和static地址的不同取法
  VIP_INTERFACE=$(ip addr show | grep ${VIP_PREFIX} | awk -F 'dynamic' '{print $2}' | head -1)
  [ -z ${VIP_INTERFACE} ] && VIP_INTERFACE=$(ip addr show | grep ${VIP_PREFIX} | awk -F 'global' '{print $2}' | head -1)
  ###
  LOCAL_IP=$(ip addr show | grep ${VIP_PREFIX} | awk -F / '{print $1}' | awk -F ' ' '{print $2}' | head -1)
  MASTER_NODES=$(echo $3 | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')
  MASTER_NODES_NO_LOCAL_IP=$(echo "${MASTER_NODES}" | sed -e 's/'${LOCAL_IP}'//g')
}
 
kube::install_keepalived()
{
    kube::get_env $@
    set +e
    which keepalived > /dev/null 2>&1
    i=$?
    set -e
    if [ $i -ne 0 ]; then
        ip addr add ${KUBE_VIP}/32 dev ${VIP_INTERFACE}
        curl -L http://$HTTP_SERVER/rpms/keepalived.tar.gz > /tmp/keepalived.tar.gz
        tar zxf /tmp/keepalived.tar.gz -C /tmp
        yum localinstall -y  /tmp/keepalived/*.rpm
        rm -rf /tmp/keepalived*
        systemctl enable keepalived.service && systemctl start keepalived.service
        kube::config_keepalived
    fi
}
 
kube::config_keepalived()
{
  echo "gen keepalived configuration"
cat <<EOF >/etc/keepalived/keepalived.conf
global_defs {
   router_id LVS_k8s
}
 
vrrp_script CheckK8sMaster {
    script "curl http://127.0.0.1:8080"
    interval 3
    timeout 9
    fall 2
    rise 2
}
 
vrrp_instance VI_1 {
    state ${HA_STATE}
    interface ${VIP_INTERFACE}
    virtual_router_id 61
    priority ${HA_PRIORITY}
    advert_int 1
    mcast_src_ip ${LOCAL_IP}
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 378378
    }
    unicast_peer {
        ${MASTER_NODES_NO_LOCAL_IP}
    }
    virtual_ipaddress {
        ${KUBE_VIP}
    }
    track_script {
        CheckK8sMaster
    }
}
 
EOF
  modprobe ip_vs
  systemctl daemon-reload && systemctl restart keepalived.service
}
 
kube::save_master_ip()
{
    set +e
    # 应该从$2里拿到etcd集群的 --endpoints, 这里默认走的127.0.0.1:2379
    [ ${KUBE_HA} == true ] && etcdctl mk ha_master ${LOCAL_IP}
    set -e
}
 
kube::copy_master_config()
{
    local master_ip=$(etcdctl get ha_master)
    mkdir -p /etc/kubernetes
    scp -r root@${master_ip}:/etc/kubernetes/* /etc/kubernetes/
    systemctl start kubelet
}
 
kube::set_label()
{
  until kubectl get no | grep `hostname`; do sleep 1; done
  kubectl label node `hostname` kubeadm.alpha.kubernetes.io/role=master
}
 
kube::master_up()
{
    shift
 
    kube::install_docker
 
    kube::load_images
 
    kube::install_bin
 
    [ ${KUBE_HA} == true ] && kube::install_keepalived "MASTER" $@
 
    # 存储master_ip，master02和master03需要用这个信息来copy配置
    kube::save_master_ip
 
    # 这里一定要带上--pod-network-cidr参数，不然后面的flannel网络会出问题
    kubeadm init --use-kubernetes-version=v1.5.1 --pod-network-cidr=10.244.0.0/16 $@
 
    # 使master节点可以被调度
    # kubectl taint nodes --all dedicated-
 
    echo -e "\033[32m 注意记录下token信息，node加入集群时需要使用！\033[0m"
 
    # install flannel network
    kubectl apply -f http://$HTTP_SERVER/network/kube-flannel.yaml --namespace=kube-system
 
    # show pods
    kubectl get pod --all-namespaces
}
 
kube::replica_up()
{
    shift
 
    kube::install_docker
 
    kube::load_images
 
    kube::install_bin
 
    kube::install_keepalived "BACKUP" $@
 
    kube::copy_master_config
 
    kube::set_label
 
}
 
kube::node_up()
{
    kube::install_docker
 
    kube::load_images
 
    kube::install_bin
 
    kube::disable_static_pod
 
    kubeadm join $@
}
 
kube::tear_down()
{
    systemctl stop kubelet.service
    docker ps -aq|xargs -I '{}' docker stop {}
    docker ps -aq|xargs -I '{}' docker rm {}
    df |grep /var/lib/kubelet|awk '{ print $6 }'|xargs -I '{}' umount {}
    rm -rf /var/lib/kubelet && rm -rf /etc/kubernetes/ && rm -rf /var/lib/etcd
    yum remove -y kubectl kubeadm kubelet kubernetes-cni
    if [ ${KUBE_HA} == true ]
    then
      yum remove -y keepalived
      rm -rf /etc/keepalived/keepalived.conf
    fi
    rm -rf /var/lib/cni
    ip link del cni0
}
 
main()
{
    case $1 in
    "m" | "master" )
        kube::master_up $@
        ;;
    "r" | "replica" )
        kube::replica_up $@
        ;;
    "j" | "join" )
        shift
        kube::node_up $@
        ;;
    "d" | "down" )
        kube::tear_down
        ;;
    *)
        echo "usage: $0 m[master] | r[replica] | j[join] token | d[down] "
        echo "       $0 master to setup master "
        echo "       $0 replica to setup replica master "
        echo "       $0 join   to join master with token "
        echo "       $0 down   to tear all down ,inlude all data! so becarefull"
        echo "       unkown command $0 $@"
        ;;
    esac
}
 
main $@

脚本使用方法

1、在一台单独的server上启动一个http-server，用来存放image和rpm包等文件，脚本会从此处拉取文件。

# nohup python -m SimpleHTTPServer &
Serving HTTP on 0.0.0.0 port 8000 ...

这是我的http-server地址：http://172.16.200.90:8000/

# tree 
.
├── etcd
│   ├── deploy-etcd.sh
│   └── temp-etcd
│       ├── etcd
│       └── etcdctl
├── images
│   ├── dnsmasq-metrics-amd64_1.0.tar
│   ├── etcd_v3.0.15.tar
│   ├── exechealthz-amd64_1.2.tar
│   ├── flannel-git_0.7.0.tar
│   ├── kube-apiserver-amd64_v1.5.1.tar
│   ├── kube-controller-manager-amd64_v1.5.1.tar
│   ├── kube-discovery-amd64_1.0.tar
│   ├── kubedns-amd64_1.9.tar
│   ├── kube-dnsmasq-amd64_1.4.tar
│   ├── kube-proxy-amd64_v1.5.1.tar
│   ├── kubernetes-dashboard-amd64.tar
│   ├── kube-scheduler-amd64_v1.5.1.tar
│   └── pause-amd64_3.0.tar
├── k8s-deploy.sh
├── network
│   └── kube-flannel.yaml
├── nohup.out
├── README.md
└── rpms
    ├── docker.tar.gz
    ├── haproxy.tar.gz
    ├── k8s.tar.gz
    └── keepalived.tar.gz

2、部署master01，在master01上执行脚本。

# curl -L http://172.60.0.43:8000/k8s-deploy.sh | bash -s master --api-advertise-addresses=172.60.0.87 --external-etcd-endpoints=http://172.60.0.226:2379,http://172.60.0.86:2379,http://172.60.0.106:2379

172.60.0.43:8000 	是http-server

--api-advertise-addresses 	 是vip地址

--external-etcd-endpoints 	 是etcd集群的地址

记录下你的 token 输出，node节点加入集群时需要使用该token。

3、部署master02和master03。这里需要分别设置两个节点与master01的ssh互信。然后分别在master02和master03上执行脚本。完成后会自动和master01组成冗余。

# curl -L http://172.60.0.43:8000/k8s-deploy.sh | bash -s replica --api-advertise-addresses=172.60.0.87 --external-etcd-endpoints=http://172.60.0.226:2379,http://172.60.0.86:2379,http://172.60.0.106:2379

上面步骤完成之后，就实现了master节点的高可用。

4、部署node。在每个node上分别执行脚本就即可。

# curl -L http://172.60.0.43:8000/k8s-deploy.sh |  bash -s join --token=3635d0.6d0caa140b219bc0 172.60.0.87   		这里的token就是部署master01完成后记录下的token

加入集群时，这里有可能会报 refuse 错误，将 kube-discovery 扩容到三个副本即可。

# kubectl scale deployment --replicas 3 kube-discovery -n kube-system 

5、完成后就得到了一个完整的高可用集群。

# kubectl get node
NAME             STATUS         AGE
kube-node02      Ready          22h
kuber-master01   Ready,master   23h
kuber-master02   Ready,master   23h
kuber-master03   Ready,master   23h
kuber-node01     Ready          23h
kuber-node03     Ready          23h
kuber-node04     Ready          23h

# kubectl get pod --all-namespaces -o wide
NAMESPACE     NAME                                     READY     STATUS    RESTARTS   AGE       IP             NODE
kube-system   dummy-2088944543-191tw                   1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-apiserver-kuber-master01            1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-apiserver-kuber-master02            1/1       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-apiserver-kuber-master03            1/1       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kube-controller-manager-kuber-master01   1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-controller-manager-kuber-master02   1/1       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-controller-manager-kuber-master03   1/1       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kube-discovery-1769846148-53vs5          1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-discovery-1769846148-m18d0          1/1       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kube-discovery-1769846148-tf0m9          1/1       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-dns-2924299975-80fnn                4/4       Running   0          1d        10.244.0.2     kuber-master01
kube-system   kube-flannel-ds-51db4                    2/2       Running   0          23h       172.60.0.87    kuber-master01
kube-system   kube-flannel-ds-gsn3m                    2/2       Running   4          23h       172.60.0.227   kuber-node01
kube-system   kube-flannel-ds-httmj                    2/2       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-flannel-ds-tq4xn                    2/2       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kube-flannel-ds-w206v                    2/2       Running   1          23h       172.60.0.44    kuber-node03
kube-system   kube-flannel-ds-x1qv3                    2/2       Running   0          22h       172.60.0.228   kube-node02
kube-system   kube-flannel-ds-xzn9l                    2/2       Running   1          23h       172.60.0.46    kuber-node04
kube-system   kube-proxy-67m5m                         1/1       Running   0          23h       172.60.0.44    kuber-node03
kube-system   kube-proxy-6gkm4                         1/1       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-proxy-7l8c8                         1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-proxy-mb650                         1/1       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kube-proxy-nb24x                         1/1       Running   0          23h       172.60.0.46    kuber-node04
kube-system   kube-proxy-qlwhj                         1/1       Running   0          22h       172.60.0.228   kube-node02
kube-system   kube-proxy-rhwrw                         1/1       Running   1          23h       172.60.0.227   kuber-node01
kube-system   kube-scheduler-kuber-master01            1/1       Running   0          1d        172.60.0.87    kuber-master01
kube-system   kube-scheduler-kuber-master02            1/1       Running   0          23h       172.60.0.86    kuber-master02
kube-system   kube-scheduler-kuber-master03            1/1       Running   0          23h       172.60.0.87    kuber-master03
kube-system   kubernetes-dashboard-3000605155-s5f7t    1/1       Running   0          22h       10.244.6.2     kuber-node01