kubernetes集群之etcd 集群
三个主节点一个node 节点做免密登录
[root@localhost ~]# hostnamectl set-hostname master-1 && exec bash [root@master-1 ~]# [root@master-1 ~]# [root@master-1 ~]# [root@master-1 ~]# ssh-key ssh-keygen ssh-keyscan [root@master-1 ~]# ssh-keygen Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Created directory '/root/.ssh'. Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: SHA256:Nj2i/OprB0z5pTv8vHqiHAMCqPqUl7y6B6ZWRBssSfk root@master-1 The key's randomart image is: +---[RSA 2048]----+ |..+ | |.+ + | |o + o . | |.. E o .. | |. o .o .Soo | |. o= +oooo . | |.o+.+ =o . | |.+ ..o.+*.. | |. ++.o*=o*+. | +----[SHA256]-----+ [root@master-1 ~]# vim /etc/hosts [root@master-1 ~]# [root@master-1 ~]# [root@master-1 ~]# [root@master-1 ~]# ssh-copy-id anaconda-ks.cfg .bash_history .bash_logout .bash_profile .bashrc .cshrc .ssh/ .tcshrc .viminfo [root@master-1 ~]# ssh-copy-id anaconda-ks.cfg .bash_history .bash_logout .bash_profile .bashrc .cshrc .ssh/ .tcshrc .viminfo [root@master-1 ~]# ssh-copy-id master-1 /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" The authenticity of host 'master-1 (192.168.10.29)' can't be established. ECDSA key fingerprint is SHA256:T9yZYCrcVc0EtAUoRLsxgWbeKAM+x3Can+rpn9MjpnM. ECDSA key fingerprint is MD5:42:ea:9c:5d:f9:96:02:df:d8:1d:ee:c4:7c:61:f5:ad. Are you sure you want to continue connecting (yes/no)? yes /usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@master-1's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'master-1'" and check to make sure that only the key(s) you wanted were added. [root@master-1 ~]# ssh-copy-id master-2 /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" The authenticity of host 'master-2 (192.168.10.30)' can't be established. ECDSA key fingerprint is SHA256:T9yZYCrcVc0EtAUoRLsxgWbeKAM+x3Can+rpn9MjpnM. ECDSA key fingerprint is MD5:42:ea:9c:5d:f9:96:02:df:d8:1d:ee:c4:7c:61:f5:ad. Are you sure you want to continue connecting (yes/no)? yes /usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@master-2's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'master-2'" and check to make sure that only the key(s) you wanted were added. [root@master-1 ~]# ssh-copy-id master-3 /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" The authenticity of host 'master-3 (192.168.10.31)' can't be established. ECDSA key fingerprint is SHA256:T9yZYCrcVc0EtAUoRLsxgWbeKAM+x3Can+rpn9MjpnM. ECDSA key fingerprint is MD5:42:ea:9c:5d:f9:96:02:df:d8:1d:ee:c4:7c:61:f5:ad. Are you sure you want to continue connecting (yes/no)? yes /usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@master-3's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'master-3'" and check to make sure that only the key(s) you wanted were added. [root@master-1 ~]# scp /etc/hosts master-2:/etc/hosts hosts 100% 256 67.0KB/s 00:00 [root@master-1 ~]# scp /etc/hosts master-3:/etc/hosts hosts [root@master-1 ~]# scp /etc/hosts node-1:/etc/hosts ^[[AThe authenticity of host 'node-1 (192.168.10.32)' can't be established. ECDSA key fingerprint is SHA256:T9yZYCrcVc0EtAUoRLsxgWbeKAM+x3Can+rpn9MjpnM. ECDSA key fingerprint is MD5:42:ea:9c:5d:f9:96:02:df:d8:1d:ee:c4:7c:61:f5:ad. Are you sure you want to continue connecting (yes/no)? yes Warning: Permanently added 'node-1,192.168.10.32' (ECDSA) to the list of known hosts. root@node-1's password: hosts 100% 256 54.2KB/s 00:00 [root@master-1 ~]# ssh-copy-id node-1 /usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub" /usr/bin/ssh-copy-id: INFO: attempting to log in with the new key(s), to filter out any that are already installed /usr/bin/ssh-copy-id: INFO: 1 key(s) remain to be installed -- if you are prompted now it is to install the new keys root@node-1's password: Number of key(s) added: 1 Now try logging into the machine, with: "ssh 'node-1'" and check to make sure that only the key(s) you wanted were added.
主机名配置（‘所有节点）
cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.10.29 master-1 192.168.10.30 master-2 192.168.10.31 master-3 192.168.10.32 node-1
关闭与禁用交换分区(所有节点)
swapoff -a vim /etc/fstab # # /etc/fstab # Created by anaconda on Sun Feb 7 10:14:45 2021 # # Accessible filesystems, by reference, are maintained under '/dev/disk' # See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info # /dev/mapper/centos-root / xfs defaults 0 0 UUID=ec65c557-715f-4f2b-beae-ec564c71b66b /boot xfs defaults 0 0 #/dev/mapper/centos-swap swap swap defaults 0 0
禁用停止firewalld.service (所有节点)
systemctl stop firewalld.service systemctl disable firewalld.service
关闭selinux(所有节点)
setenforce 0（临时关掉） vim /etc/selinux/config # This file controls the state of SELinux on the system. # SELINUX= can take one of these three values: # enforcing - SELinux security policy is enforced. # permissive - SELinux prints warnings instead of enforcing. # disabled - No SELinux policy is loaded. SELINUX=disabled # SELINUXTYPE= can take one of three values: # targeted - Targeted processes are protected, # minimum - Modification of targeted policy. Only selected processes are protected. # mls - Multi Level Security protection. SELINUXTYPE=targeted
修改内核参数；加载内核模块
modprobe br_netfilter #验证模块是否加载成功： lsmod |grep br_netfilter #修改内核参数 cat > /etc/sysctl.d/k8s.conf <<EOF net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 EOF #使刚才修改的内核参数生效 sysctl -p /etc/sysctl.d/k8s.conf echo "modprobe br_netfilter" >> /etc/profile
在运行时配置内核参数
-p 从指定的文件加载系统参数，如不指定即从/etc/sysctl.conf中加载
问题2：为什么要执行modprobe br_netfilter？
修改/etc/sysctl.d/k8s.conf文件，增加如下三行参数：
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
sysctl -p /etc/sysctl.d/k8s.conf出现报错：
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
解决方法：
modprobe br_netfilter
问题3：为什么开启net.bridge.bridge-nf-call-iptables内核参数？
在centos下安装docker，执行docker info出现如下警告：
WARNING: bridge-nf-call-iptables is disabled
WARNING: bridge-nf-call-ip6tables is disabled
解决办法：
vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
问题4：为什么要开启net.ipv4.ip_forward = 1参数？
kubeadm初始化k8s如果报错：
就表示没有开启ip_forward，需要开启。
net.ipv4.ip_forward是数据包转发：
出于安全考虑，Linux系统默认是禁止数据包转发的。所谓转发即当主机拥有多于一块的网卡时，其中一块收到数据包，根据数据包的目的ip地址将数据包发往本机另一块网卡，该网卡根据路由表继续发送数据包。这通常是路由器所要实现的功能。
要让Linux系统具有路由转发功能，需要配置一个Linux的内核参数net.ipv4.ip_forward。这个参数指定了Linux系统当前对路由转发功能的支持情况；其值为0时表示禁止进行IP转发；如果是1,则说明IP转发功能已经打开。
docker安装
https://www.cnblogs.com/rdchenxi/p/10381631.html
镜像加速器配置
curl -sSL https://get.daocloud.io/daotools/set_mirror.sh | sh -s http://f1361db2.m.daocloud.io
修改docker 启动引擎
/etc/docker/daemon.json << 'EOF'
{
"registry-mirrors":["https://rsbud4vc.mirror.aliyuncs.com","https://registry.docker-cn.com","https://docker.mirrors.ustc.edu.cn","https://dockerhub.azk8s.cn","http://hub-mirror.c.163.com","http://qtid6917.mirror.aliyuncs.com", "https://rncxm540.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
systemctl daemon-reload
systemctl restart docker
systemctl status docker
开机启动
systemctl enable docker.service
配置时间同步
yum -y install ntpdate ntpdate time.windows.com #把时间同步做成计划任务 crontab -e * */1 * * * /usr/sbin/ntpdate time.windows.com #重启crond服务 service crond restart
安装iptables 工具服务
#安装iptables yum install iptables-services -y #禁用iptables service iptables stop && systemctl disable iptables #清空防火墙规则 iptables -F
开启ipvs
cd /etc/sysconfig/modules/
vim ipvs.modules
#!/bin/bash
ipvs_modules="ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_nq ip_vs_sed ip_vs_ftp nf_conntrack"
for kernel_module in ${ipvs_modules}; do
/sbin/modinfo -F filename ${kernel_module} > /dev/null 2>&1
if [ 0 -eq 0 ]; then
/sbin/modprobe ${kernel_module}
fi
done
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep ip_vs
ip_vs_ftp 13079 0
ip_vs_sed 12519 0
ip_vs_nq 12516 0
ip_vs_sh 12688 0
ip_vs_dh 12688 0
ip_vs_lblcr 12922 0
ip_vs_lblc 12819 0
ip_vs_wrr 12697 0
ip_vs_rr 12600 0
ip_vs_wlc 12519 0
ip_vs_lc 12516 0
ip_vs 145497 22 ip_vs_dh,ip_vs_lc,ip_vs_nq,ip_vs_rr,ip_vs_sh,ip_vs_ftp,ip_vs_sed,ip_vs_wlc,ip_vs_wrr,ip_vs_lblcr,ip_vs_lblc
nf_nat 26787 3 ip_vs_ftp,nf_nat_ipv4,nf_nat_masquerade_ipv4
nf_conntrack 133095 7 ip_vs,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_ipv4
libcrc32c 12644 4 xfs,ip_vs,nf_nat,nf_conntrack
安装基础包
yum install -y yum-utils device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel python-devel epel-release openssh-server socat ipvsadm conntrack ntpdate telnet rsync
下载生成证书的工具
cd /usr/bin/ wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 chmod +x cfssl* mv cfssl_linux-amd64 cfssl mv cfssljson_linux-amd64 cfssljson mv cfssl-certinfo_linux-amd64 cfssl-certinfo
生成etcd 集群证书
#创建配置文件和证书文件存放目录 [root@master* ~]# mkdir -p /etc/etcd [root@master*~]# mkdir -p /etc/etcd/ssl # 创建签署证书目录 mkdir /data/work -p [root@master-1 bin]# cd /data/work/ [root@master-1 work]# ls
生成CA证书请求文件
vim ca-csr.json
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}
],
"ca": {
"expiry": "87600h"
}
}
生成CA证书
[root@master-1 work]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca 2021/09/22 19:17:27 [INFO] generating a new CA key and certificate from CSR 2021/09/22 19:17:27 [INFO] generate received request 2021/09/22 19:17:27 [INFO] received CSR 2021/09/22 19:17:27 [INFO] generating key: rsa-2048 2021/09/22 19:17:27 [INFO] encoded CSR 2021/09/22 19:17:27 [INFO] signed certificate with serial number 170756754160921488966564476114626083230418034400 [root@master-1 work]# ls ca.csr ca-csr.json ca-key.pem ca.pem
生成ca证书文件
vim ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
生成etcd证书
#配置etcd证书请求，hosts的ip变成自己etcd所在节点的ip；hosts字段中IP为所有etcd节点的集群内部通信IP，可以预留几个，做扩容用。
vim etcd-csr.json
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.10.29",
"192.168.10.30",
"192.168.10.31",
"192.168.10.28" #漂流IP
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "Hubei",
"L": "Wuhan",
"O": "k8s",
"OU": "system"
}]
}
签发etcd证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
查看
[root@master-1 work]# ls etcd* etcd.csr etcd-csr.json etcd-key.pem etcd.pem
下载etcd包
wget https://github.com/etcd-io/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz
解压
tar xf etcd-v3.5.0-linux-amd64.tar.gz cd etcd-v3.5.0-linux-amd64/
拷贝执行目录与分发
[root@master-1 etcd-v3.5.0-linux-amd64]# cp etcd* /usr/local/bin/ 您在 /var/spool/mail/root 中有新邮件 [root@master-1 etcd-v3.5.0-linux-amd64]# scp /usr/local/bin/etcd* master-2:/usr/local/bin/ etcd 100% 22MB 117.7MB/s 00:00 etcdctl 100% 17MB 133.4MB/s 00:00 etcdutl 100% 15MB 139.5MB/s 00:00 [root@master-1 etcd-v3.5.0-linux-amd64]# scp /usr/local/bin/etcd* master-3:/usr/local/bin/ etcd 100% 22MB 95.8MB/s 00:00 etcdctl 100% 17MB 133.5MB/s 00:00 etcdutl 100% 15MB 132.3MB/s 00:00
创建etcd 配置文件
[root@master-1 etcd-v3.5.0-linux-amd64]# vim /etc/etcd/etcd.conf #[Member] ETCD_NAME="etcd1" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.29:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.29:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.29:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.29:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.10.29:2380,etcd2=https://192.168.10.30:2380,etcd3=https://192.168.10.31:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"
创建启动文件
[Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target [Service] Type=notify EnvironmentFile=-/etc/etcd/etcd.conf WorkingDirectory=/var/lib/etcd/ ExecStart=/usr/local/bin/etcd \ --cert-file=/etc/etcd/ssl/etcd.pem \ --key-file=/etc/etcd/ssl/etcd-key.pem \ --trusted-ca-file=/etc/etcd/ssl/ca.pem \ --peer-cert-file=/etc/etcd/ssl/etcd.pem \ --peer-key-file=/etc/etcd/ssl/etcd-key.pem \ --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem \ --peer-client-cert-auth \ --client-cert-auth Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target
创建数据目录（所有节点）
mkdir -p /var/lib/etcd/default.etcd
分发配置文件与启动文件
[root@master-1 work]# cp etcd*.pem /etc/etcd/ssl/ [root@master-1 work]# cp ca*.pem /etc/etcd/ssl/ [root@master-1 work]# cp etcd.service /usr/lib/systemd/system/ [root@master-1 etcd-v3.5.0-linux-amd64]# scp -r /etc/etcd master-2:/etc/ ca-key.pem 100% 1679 325.5KB/s 00:00 ca.pem 100% 1346 457.1KB/s 00:00 etcd-key.pem 100% 1679 930.8KB/s 00:00 etcd.pem 100% 1432 1.0MB/s 00:00 etcd.conf 100% 527 441.4KB/s 00:00 您在 /var/spool/mail/root 中有新邮件 [root@master-1 etcd-v3.5.0-linux-amd64]# scp -r /etc/etcd master-3:/etc/ ca-key.pem 100% 1679 736.3KB/s 00:00 ca.pem 100% 1346 1.7MB/s 00:00 etcd-key.pem 100% 1679 1.8MB/s 00:00 etcd.pem 100% 1432 2.2MB/s 00:00 etcd.conf 100% 527 957.9KB/s 00:00 [root@master-1 etcd-v3.5.0-linux-amd64]# scp etcd.service master-2:/usr/lib/systemd/system/ etcd.service 100% 634 200.2KB/s 00:00 [root@master-1 etcd-v3.5.0-linux-amd64]# scp etcd.service master-3:/usr/lib/systemd/system/
修改其他俩节点配置
#[Member] ETCD_NAME="etcd2" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.30:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.30:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.30:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.30:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.10.29:2380,etcd2=https://192.168.10.30:2380,etcd3=https://192.168.10.31:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new" #[Member] ETCD_NAME="etcd3" ETCD_DATA_DIR="/var/lib/etcd/default.etcd" ETCD_LISTEN_PEER_URLS="https://192.168.10.31:2380" ETCD_LISTEN_CLIENT_URLS="https://192.168.10.31:2379,http://127.0.0.1:2379" #[Clustering] ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.10.31:2380" ETCD_ADVERTISE_CLIENT_URLS="https://192.168.10.31:2379" ETCD_INITIAL_CLUSTER="etcd1=https://192.168.10.29:2380,etcd2=https://192.168.10.30:2380,etcd3=https://192.168.10.31:2380" ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" ETCD_INITIAL_CLUSTER_STATE="new"
启动集群
systemctl daemon-reload systemctl enable etcd.service systemctl start etcd.service systemctl status etcd
查看集群状态
[root@master-1 etcd-v3.5.0-linux-amd64]# /usr/local/bin/etcdctl --write-out=table --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://192.168.10.29 :2379,https://192.168.10.30:2379,https://192.168.10.31:2379 endpoint health
+----------------------------+--------+-------------+-------+ | ENDPOINT | HEALTH | TOOK | ERROR | +----------------------------+--------+-------------+-------+ | https://192.168.10.30:2379 | true | 8.265488ms | | | https://192.168.10.31:2379 | true | 10.124353ms | | | https://192.168.10.29:2379 | true | 8.329181ms | | +----------------------------+--------+-------------+-------+
草都可以从石头缝隙中长出来更可况你呢
