ansible方式安装二进制k8s

一、主机清单
192.168.80.100   localhost7A.localdomain        harbor        CentOS 7.7
192.168.80.110   localhost7B.localdomain        ansible        CentOS 7.7
192.168.80.120   localhost7C.localdomain     master        CentOS 7.7
192.168.80.130   localhost7D.localdomain     master        CentOS 7.7
192.168.80.140   localhost7E.localdomain     master        CentOS 7.7
192.168.80.150   localhost7F.localdomain     node1        CentOS 7.7
192.168.80.160   localhost7G.localdomain     node2        CentOS 7.7
192.168.80.170   localhost7H.localdomain     node3        CentOS 7.7
192.168.80.180   localhost7I.localdomain     etcd        CentOS 7.7
192.168.80.190   localhost7J.localdomain     etcd        CentOS 7.7
192.168.80.200   localhost7H.localdomain     etcd        CentOS 7.7



二、所有服务器时间同步，设置YUM源
ntpdate   time1.aliyun.com && hwclock  -w

所有服务器安装python环境，ansible依赖。
# apt update
# apt-get install python2.7 –y
# ln -s /usr/bin/python2.7 /usr/bin/python

源安装包，后续添加各节点需要依赖
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo




三、部署 harbor 设置HTTPS 登录 harbor.zzhz.com 

1.同步时间服务 并关闭防火墙和selinux
ntpdate   time1.aliyun.com && hwclock  -w

2.下载YUM源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo


3.安装docker  docker-compose
yum list docker-ce --showduplicates
yum install docker-ce-19.03.15-3.el7  docker-ce-cli-19.03.15-3.el7
yum install docker-compose
systemctl enable docker  systemctl start docker

4.下载harbor 解压
https://github.com/goharbor/harbor/releases/download/v1.7.6/harbor-offline-installer-v1.7.6
tar xvf harbor-offline-installer-v1.7.6.tgz 
ln -sv /usr/local/src/harbor   /usr/local/
cd /usr/local/harbor/



5.生成证书
# mkdir /usr/local/harbor/certs/ -p 
# cd  /usr/local/harbor/certs/
# openssl genrsa -out harbor-ca.key #生成私有key 
# openssl req -x509 -new -nodes -key harbor-ca.key  -subj "/CN=harbor.zzhz.com"  -days 7120  -out harbor-ca.crt #签证 

6.设置配置文件
# vim harbor.cfg 
hostname = harbor.magedu.net 
ui_url_protocol = https 
ssl_cert = /usr/local/harbor/certs/harbor-ca.crt 
ssl_cert_key = /usr/local/harbor/certs/harbor-ca.key
harbor_admin_password = Harbor12345
# ./install.sh


7.在ansible主机创建文件
# mkdir /etc/docker/certs.d/harbor.zzhz.com -p 

8.复制文件到ansible
#  scp /usr/local/harbor/certs/harbor-ca.crt   192.168.80.110:/etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt

9./etc/hosts解析登录测试
# docker login harbor.zzhz.com
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded




四、在ansible主机

1.在ansible控制端配置免密码登录，安装ansible  python  sshpass  git

[root@localhost7B ~]#yum  install python2.7 
[root@localhost7B ~]#ln -s /usr/bin/python2.7 /usr/bin/python 

2.#sshpass同步公钥到各k8s服务器的工具
[root@localhost7B ~]# yum  install ansible  sshpass 
[root@localhost7B ~]# ssh-keygen #生成密钥对


3..设置hosts  /etc/hosts解析
192.168.80.100   localhost7A.localdomain  harbor.zzhz.com
192.168.80.110   localhost7B.localdomain
192.168.80.120   localhost7C.localdomain
192.168.80.130   localhost7D.localdomain 
192.168.80.140   localhost7E.localdomain
192.168.80.150   localhost7F.localdomain
192.168.80.160   localhost7G.localdomain
192.168.80.170   localhost7H.localdomain
192.168.80.180   localhost7I.localdomain
192.168.80.190   localhost7J.localdomain
192.168.80.200   localhost7J.localdomain

3.复制  证书 公钥  hosts  DNS文件,
#!/bin/bash 
#目标主机列表 
IP="192.168.80.100
    192.168.80.110
    192.168.80.120
    192.168.80.130
    192.168.80.140
    192.168.80.150
    192.168.80.160
    192.168.80.170
    192.168.80.180
    192.168.80.190
    192.168.80.200 "
for node in ${IP};do
  sshpass -p password1! ssh-copy-id ${node} -o StrictHostKeyChecking=no
  if [ $? -eq 0 ];then
    
    echo "${node} 秘钥copy完成"
    echo "${node} 秘钥copy完成,准备环境初始化....."

    ssh ${node} "mkdir /etc/docker/certs.d/harbor.zzhz.com -p"
    echo "Harbor 证书目录创建成功!" 
    scp /etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt ${node}:/etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt
    echo "Harbor 证书拷贝成功!"   
    #scp -r /root/.docker ${node}:/root/
    #echo "Harbor 认证文件拷贝完成!" 

    scp /etc/hosts ${node}:/etc/hosts
    echo "host 文件拷贝完成" 

    scp -r /etc/resolv.conf ${node}:/etc/
  else
    echo "${node} 秘钥copy失败" 
fi

done





4.下载工具脚本easzup，脚本中定义各软件的版本。
https://github.com/easzlab/kubeasz/releases/download/2.2.0/easzup

编辑脚本内容。定义软件版本
cat   easzup
# default version, can be overridden by cmd line options
export DOCKER_VER=19.03.8    #这个
export KUBEASZ_VER=2.2.0
export K8S_BIN_VER=v1.17.4     #这个
export EXT_BIN_VER=0.4.0
export SYS_PKG_VER=0.3.3

chmod +x ./easzup
./easzup -D                #开始下载
ll /etc/ansible/down/   #下载完成目录：

rm -rf /etc/ansible/*     #删除ansible已有文件
mv kubeasz/* /etc/ansible/




五、安装k8s
1.必要配置：
cd /etc/ansible && cp example/hosts.multi-node hosts, 然后实际情况修改此hosts文件

2.修改文件
# 'etcd' cluster should have odd member(s) (1,3,5,...)
# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster
# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点
[etcd]
192.168.80.170 NODE_NAME=etcd1
192.168.80.180 NODE_NAME=etcd2
#192.168.80.190 NODE_NAME=etcd3

# master node(s)
[kube-master]
192.168.80.120 NEW_MASTER=yes 
192.168.80.130
#192.168.80.140

# work node(s)
[kube-node]
192.168.80.150 NEW_NODE=yes 
192.168.80.160
#192.168.80.170

# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one
# 'SELF_SIGNED_CERT': 'no' you need put files of certificates named harbor.pem and harbor-key.pem in directory 'down'
[harbor]
#192.168.80.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no SELF_SIGNED_CERT=yes

# [optional] loadbalance for accessing k8s from outside
#负载均衡(目前已支持多于2节点，一般2节点就够了) 安装 haproxy+keepalived(如果没有部署，默认使用LVS)
[ex-lb]
192.168.80.110 LB_ROLE=master EX_APISERVER_VIP=192.168.80.222 EX_APISERVER_PORT=6443
#192.168.80.211 LB_ROLE=backup EX_APISERVER_VIP=192.168.80.188 EX_APISERVER_PORT=6443

# [optional] ntp server for the cluster
[chrony]
#192.168.80.1

[all:vars]
# --------- Main Variables ---------------
# Cluster container-runtime supported: docker, containerd
CONTAINER_RUNTIME="docker"

# 集群网络插件，目前支持calico, flannel, kube-router, cilium
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="flannel"

kube proxy的服务代理模式：“iptables”或“IPV”
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"

# 服务网段 (Service CIDR），注意不要与内网已有网段冲突
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="172.28.0.0/16"

# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.20.0.0/16"

# 服务端口范围
# NodePort Range
NODE_PORT_RANGE="30000-60000"

#集群 DNS 域名 
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="magedu.local."

#默认二进制文件目录
# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/usr/bin"

#证书目录
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"

#部署目录，即 ansible 工作目录，建议不要修改
# Deploy Directory (kubeasz workspace)
base_dir="/etc/ansible"



3.安装各节点
#环境初始化
ansible-playbook 01.prepare.yml
#部署etcd集群
ansible-playbook 02.etcd.yml
#部署docker
ansible-playbook 03.docker.yml
#部署master
ansible-playbook 04.kube-master.yml
#部署node
ansible-playbook 05.kube-node.yml
#部署网络服务flannel,实际中这步没操作。
ansible-playbook 06.network.yml

增加 etcd  节点：   $ easzctl add-etcd 192.168.80.140(注意：增加 etcd 还需要根据提示输入集群内唯一的 NODE_NAME,在host文件定义)
增加 master节点：   $ easzctl add-master 192.168.80.170
增加 node  节点：   $ easzctl add-node 192.168.80.200



4.#创建pod测试夸主机网络通信是否正常(域名无法ping通，是DNS没有设置)
kubectl run net-test1 --image=alpine --replicas=4 sleep 360000 




六、k8s 升级小版本
方式一：下载二进制k8s版本解压， stop节点服务，复制server/bin/5个配置文件到/usr/bin/下，重启服务。
方式一：下载二进制k8s版本解压，复制server/bin/kube*开头的二进制文件到/etc/kubeasz/bin，控制端执行easzctl  upgrade。

节点版本升级说明
node:kubelet kube-proxy  kubectl
master: kube-apiserver  kube-controller-manager kubelet kube-proxy  kube-scheduler  kubectl 

[root@localhost7F ~]# ps aux  | grep kube
[root@localhost7F ~]# systemctl  stop  kube-apiserver kube-controller-manager kubelet kube-proxy kube-scheduler







/ansible/roles/kube-node/templates/kubelet.service 定义的镜像变量
设置node节点镜像下载点ansible/roles/kube-node/defaults/mail.yaml
# 基础容器镜像 SANDBOX_IMAGE: "harbor.exmple.demo/baseimages/pause-amd64:3.1"


node节点采用的是haproxy方式访问master   启动脚本中定义的配置文件  /etc/kubernets/kubelet.kubeconfig