一、主机清单
192.168.80.100 localhost7A.localdomain harbor CentOS 7.7
192.168.80.110 localhost7B.localdomain ansible CentOS 7.7
192.168.80.120 localhost7C.localdomain master CentOS 7.7
192.168.80.130 localhost7D.localdomain master CentOS 7.7
192.168.80.140 localhost7E.localdomain master CentOS 7.7
192.168.80.150 localhost7F.localdomain node1 CentOS 7.7
192.168.80.160 localhost7G.localdomain node2 CentOS 7.7
192.168.80.170 localhost7H.localdomain node3 CentOS 7.7
192.168.80.180 localhost7I.localdomain etcd CentOS 7.7
192.168.80.190 localhost7J.localdomain etcd CentOS 7.7
192.168.80.200 localhost7H.localdomain etcd CentOS 7.7
二、所有服务器时间同步，设置YUM源
ntpdate time1.aliyun.com && hwclock -w
所有服务器安装python环境，ansible依赖。
# apt update
# apt-get install python2.7 –y
# ln -s /usr/bin/python2.7 /usr/bin/python
源安装包，后续添加各节点需要依赖
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
三、部署 harbor 设置HTTPS 登录 harbor.zzhz.com
1.同步时间服务 并关闭防火墙和selinux
ntpdate time1.aliyun.com && hwclock -w
2.下载YUM源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
3.安装docker docker-compose
yum list docker-ce --showduplicates
yum install docker-ce-19.03.15-3.el7 docker-ce-cli-19.03.15-3.el7
yum install docker-compose
systemctl enable docker systemctl start docker
4.下载harbor 解压
https://github.com/goharbor/harbor/releases/download/v1.7.6/harbor-offline-installer-v1.7.6
tar xvf harbor-offline-installer-v1.7.6.tgz
ln -sv /usr/local/src/harbor /usr/local/
cd /usr/local/harbor/
5.生成证书
# mkdir /usr/local/harbor/certs/ -p
# cd /usr/local/harbor/certs/
# openssl genrsa -out harbor-ca.key #生成私有key
# openssl req -x509 -new -nodes -key harbor-ca.key -subj "/CN=harbor.zzhz.com" -days 7120 -out harbor-ca.crt #签证
6.设置配置文件
# vim harbor.cfg
hostname = harbor.magedu.net
ui_url_protocol = https
ssl_cert = /usr/local/harbor/certs/harbor-ca.crt
ssl_cert_key = /usr/local/harbor/certs/harbor-ca.key
harbor_admin_password = Harbor12345
# ./install.sh
7.在ansible主机创建文件
# mkdir /etc/docker/certs.d/harbor.zzhz.com -p
8.复制文件到ansible
# scp /usr/local/harbor/certs/harbor-ca.crt 192.168.80.110:/etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt
9./etc/hosts解析登录测试
# docker login harbor.zzhz.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
四、在ansible主机
1.在ansible控制端配置免密码登录，安装ansible python sshpass git
[root@localhost7B ~]#yum install python2.7
[root@localhost7B ~]#ln -s /usr/bin/python2.7 /usr/bin/python
2.#sshpass同步公钥到各k8s服务器的工具
[root@localhost7B ~]# yum install ansible sshpass
[root@localhost7B ~]# ssh-keygen #生成密钥对
3..设置hosts /etc/hosts解析
192.168.80.100 localhost7A.localdomain harbor.zzhz.com
192.168.80.110 localhost7B.localdomain
192.168.80.120 localhost7C.localdomain
192.168.80.130 localhost7D.localdomain
192.168.80.140 localhost7E.localdomain
192.168.80.150 localhost7F.localdomain
192.168.80.160 localhost7G.localdomain
192.168.80.170 localhost7H.localdomain
192.168.80.180 localhost7I.localdomain
192.168.80.190 localhost7J.localdomain
192.168.80.200 localhost7J.localdomain
3.复制 证书 公钥 hosts DNS文件,
#!/bin/bash
#目标主机列表
IP="192.168.80.100
192.168.80.110
192.168.80.120
192.168.80.130
192.168.80.140
192.168.80.150
192.168.80.160
192.168.80.170
192.168.80.180
192.168.80.190
192.168.80.200 "
for node in ${IP};do
sshpass -p password1! ssh-copy-id ${node} -o StrictHostKeyChecking=no
if [ $? -eq 0 ];then
echo "${node} 秘钥copy完成"
echo "${node} 秘钥copy完成,准备环境初始化....."
ssh ${node} "mkdir /etc/docker/certs.d/harbor.zzhz.com -p"
echo "Harbor 证书目录创建成功!"
scp /etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt ${node}:/etc/docker/certs.d/harbor.zzhz.com/harbor-ca.crt
echo "Harbor 证书拷贝成功!"
#scp -r /root/.docker ${node}:/root/
#echo "Harbor 认证文件拷贝完成!"
scp /etc/hosts ${node}:/etc/hosts
echo "host 文件拷贝完成"
scp -r /etc/resolv.conf ${node}:/etc/
else
echo "${node} 秘钥copy失败"
fi
done
4.下载工具脚本easzup，脚本中定义各软件的版本。
https://github.com/easzlab/kubeasz/releases/download/2.2.0/easzup
编辑脚本内容。定义软件版本
cat easzup
# default version, can be overridden by cmd line options
export DOCKER_VER=19.03.8 #这个
export KUBEASZ_VER=2.2.0
export K8S_BIN_VER=v1.17.4 #这个
export EXT_BIN_VER=0.4.0
export SYS_PKG_VER=0.3.3
chmod +x ./easzup
./easzup -D #开始下载
ll /etc/ansible/down/ #下载完成目录：
rm -rf /etc/ansible/* #删除ansible已有文件
mv kubeasz/* /etc/ansible/
五、安装k8s
1.必要配置：
cd /etc/ansible && cp example/hosts.multi-node hosts, 然后实际情况修改此hosts文件
2.修改文件
# 'etcd' cluster should have odd member(s) (1,3,5,...)
# variable 'NODE_NAME' is the distinct name of a member in 'etcd' cluster
# etcd集群请提供如下NODE_NAME，注意etcd集群必须是1,3,5,7...奇数个节点
[etcd]
192.168.80.170 NODE_NAME=etcd1
192.168.80.180 NODE_NAME=etcd2
#192.168.80.190 NODE_NAME=etcd3
# master node(s)
[kube-master]
192.168.80.120 NEW_MASTER=yes
192.168.80.130
#192.168.80.140
# work node(s)
[kube-node]
192.168.80.150 NEW_NODE=yes
192.168.80.160
#192.168.80.170
# [optional] harbor server, a private docker registry
# 'NEW_INSTALL': 'yes' to install a harbor server; 'no' to integrate with existed one
# 'SELF_SIGNED_CERT': 'no' you need put files of certificates named harbor.pem and harbor-key.pem in directory 'down'
[harbor]
#192.168.80.8 HARBOR_DOMAIN="harbor.yourdomain.com" NEW_INSTALL=no SELF_SIGNED_CERT=yes
# [optional] loadbalance for accessing k8s from outside
#负载均衡(目前已支持多于2节点，一般2节点就够了) 安装 haproxy+keepalived(如果没有部署，默认使用LVS)
[ex-lb]
192.168.80.110 LB_ROLE=master EX_APISERVER_VIP=192.168.80.222 EX_APISERVER_PORT=6443
#192.168.80.211 LB_ROLE=backup EX_APISERVER_VIP=192.168.80.188 EX_APISERVER_PORT=6443
# [optional] ntp server for the cluster
[chrony]
#192.168.80.1
[all:vars]
# --------- Main Variables ---------------
# Cluster container-runtime supported: docker, containerd
CONTAINER_RUNTIME="docker"
# 集群网络插件，目前支持calico, flannel, kube-router, cilium
# Network plugins supported: calico, flannel, kube-router, cilium, kube-ovn
CLUSTER_NETWORK="flannel"
kube proxy的服务代理模式：“iptables”或“IPV”
# Service proxy mode of kube-proxy: 'iptables' or 'ipvs'
PROXY_MODE="ipvs"
# 服务网段 (Service CIDR），注意不要与内网已有网段冲突
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="172.28.0.0/16"
# POD 网段 (Cluster CIDR），注意不要与内网已有网段冲突
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.20.0.0/16"
# 服务端口范围
# NodePort Range
NODE_PORT_RANGE="30000-60000"
#集群 DNS 域名
# Cluster DNS Domain
CLUSTER_DNS_DOMAIN="magedu.local."
#默认二进制文件目录
# -------- Additional Variables (don't change the default value right now) ---
# Binaries Directory
bin_dir="/usr/bin"
#证书目录
# CA and other components cert/key Directory
ca_dir="/etc/kubernetes/ssl"
#部署目录，即 ansible 工作目录，建议不要修改
# Deploy Directory (kubeasz workspace)
base_dir="/etc/ansible"
3.安装各节点
#环境初始化
ansible-playbook 01.prepare.yml
#部署etcd集群
ansible-playbook 02.etcd.yml
#部署docker
ansible-playbook 03.docker.yml
#部署master
ansible-playbook 04.kube-master.yml
#部署node
ansible-playbook 05.kube-node.yml
#部署网络服务flannel,实际中这步没操作。
ansible-playbook 06.network.yml
增加 etcd 节点： $ easzctl add-etcd 192.168.80.140(注意：增加 etcd 还需要根据提示输入集群内唯一的 NODE_NAME,在host文件定义)
增加 master节点： $ easzctl add-master 192.168.80.170
增加 node 节点： $ easzctl add-node 192.168.80.200
4.#创建pod测试夸主机网络通信是否正常(域名无法ping通，是DNS没有设置)
kubectl run net-test1 --image=alpine --replicas=4 sleep 360000
六、k8s 升级小版本
方式一：下载二进制k8s版本解压， stop节点服务，复制server/bin/5个配置文件到/usr/bin/下，重启服务。
方式一：下载二进制k8s版本解压，复制server/bin/kube*开头的二进制文件到/etc/kubeasz/bin，控制端执行easzctl upgrade。
节点版本升级说明
node:kubelet kube-proxy kubectl
master: kube-apiserver kube-controller-manager kubelet kube-proxy kube-scheduler kubectl
[root@localhost7F ~]# ps aux | grep kube
[root@localhost7F ~]# systemctl stop kube-apiserver kube-controller-manager kubelet kube-proxy kube-scheduler
/ansible/roles/kube-node/templates/kubelet.service 定义的镜像变量
设置node节点镜像下载点ansible/roles/kube-node/defaults/mail.yaml
# 基础容器镜像 SANDBOX_IMAGE: "harbor.exmple.demo/baseimages/pause-amd64:3.1"
node节点采用的是haproxy方式访问master 启动脚本中定义的配置文件 /etc/kubernets/kubelet.kubeconfig
