centos7 部署 k8s1.25.3 版本 (使用cri-dockerd方式安装)

1.关闭防火墙、selinux

sed -i 's/^ *SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

systemctl stop firewalld.service

systemctl disable firewalld

添加yum源

1.添加centos7 aliyun 源

mv CentOS-Base.repo CentOS-Base.repo.bak
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo

2.添加docker源

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

3. 添加k8s源

[root@k8s-master ~]# cat /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg

yum clean all
yum makecache

安装 docker
yum install -y yum-utils device-mapper-persistent-data lvm2

yum list docker-ce --showduplicates | sort -r

yum install docker-ce-20.10.0-3.el7 docker-ce-cli-20.10.0-3.el7 containerd.io

修改docker 配置文件

[root@k8s-master ~]# cat /etc/docker/daemon.json
{
"registry-mirrors": ["https://v16stybc.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"]
}

systemctl start docker
systemctl enable docker

同步时间
timedatectl set-timezone Asia/Shanghai
ntpdate ntp1.aliyun.com

关闭交换分区

sed -ri 's/.*swap.*/#&/' /etc/fstab

swapoff -a
[root@k8s-master ~]# free -m
              total        used        free      shared buff/cache   available
Mem:           8077         955        6047          20        1074        6706
Swap:             0           0           0

加载模块

1. 永久添加模块

[root@k8s-master01 ~]# vim /etc/modules-load.d/k8s.conf overlay br_netfilter 2.手动添加

modprobe br_netfilter

modprobe overlay、

3.查看添加效果

lsmod | grep overlay

[root@k8s-master01 ~]# lsmod | grep br_netfilter
br_netfilter 28672 0

修改内核参数

[root@k8s-master ~]# cat /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
vm.swappiness=0 #可选

[root@k8s-master ~]# sysctl -p /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1

sysctl --system

免密登入

[root@k8s-master01 ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
/root/.ssh/id_rsa already exists.
Overwrite (y/n)? y
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:o1K5kmFUnKRVAx3jL8ZOboNAmHO/m7ubzCgdcKOGt/U root@k8s-master01
The key's randomart image is:
+---[RSA 2048]----+
|     o===.       |
|   o +o..o       |
| + =   .        |
| .=o. o .       |
| . ++.+ S .      |
|. +.o= X o       |
| o ++o= =        |
| o .*E= .       |
|   .. X+         |
+----[SHA256]-----+

[root@k8s-master01 ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub root@k8s-node01
/usr/bin/ssh-copy-id: INFO: Source of key(s) to be installed: "/root/.ssh/id_rsa.pub"
The authenticity of host 'k8s-node01 (192.168.30.120)' can't be established.
ECDSA key fingerprint is SHA256:Q48BAO1YJ/XHgdEOJvjrwaQ6z5HdREXfzai90lWVI9s.
ECDSA key fingerprint is MD5:f8:aa:e0:b1:5a:59:2f:78:4a:1c:70:fc:f9:86:60:33.
Are you sure you want to continue connecting (yes/no)? yes

安装ipvs模块

yum -y install ipset ipvsadm
[root@localhost ~]# vim /etc/sysconfig/modules/ipvs.modules

modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack

[root@k8s-master01 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack
nf_conntrack_netlink    49152 0
nfnetlink              20480 4 nf_conntrack_netlink,nf_tables,ip_set
ip_vs_sh               16384 0
ip_vs_wrr              16384 0
ip_vs_rr               16384 0
ip_vs                 163840 6 ip_vs_rr,ip_vs_sh,ip_vs_wrr
nf_conntrack          163840 6 xt_conntrack,nf_nat,xt_nat,nf_conntrack_netlink,xt_MASQUERADE,ip_vs
nf_defrag_ipv6         24576 2 nf_conntrack,ip_vs
nf_defrag_ipv4         16384 1 nf_conntrack
libcrc32c              16384 5 nf_conntrack,nf_nat,nf_tables,xfs,ip_vs

安装cri-docker

方法一：

到下面的链接下载最新版cri-docker
https://github.com/Mirantis/cri-dockerd/releases

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.2.6/cri-dockerd-0.2.6.amd64.tgz

tar -xf cri-dockerd-0.2.6.amd64.tgz
cp cri-dockerd/cri-dockerd /usr/bin/

chmod +x /usr/bin/cri-dockerd

#配置启动文件

vim /usr/lib/systemd/system/cri-docker.service
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify

ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8

ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

StartLimitBurst=3

StartLimitInterval=60s

LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target

#生成socket文件

vim /usr/lib/systemd/system/cri-docker.socket

[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target

# 启动cri-docker并设置开机自动启动

systemctl daemon-reload ; systemctl enable cri-docker --now

systemctl is-active cri-docke

如果是多master 需要操作下面步骤

#先在master上解压出cri-docker，然后拷贝到其他节点
[root@k8s-master01 ~]# tar -zxf cri-dockerd-0.2.5.amd64.tgz
[root@k8s-master01 ~]# cp cri-dockerd/cri-dockerd /usr/bin/
[root@k8s-master01 ~/cri-dockerd]# scp /usr/bin/cri-dockerd root@k8s-node01:/usr/bin/
[root@k8s-master01 ~/cri-dockerd]# scp /usr/bin/cri-dockerd root@k8s-node02:/usr/bin/
#创建cri-docker启动文件，然后拷贝到其他节点
[root@k8s-master01 ~]# cat /usr/lib/systemd/system/cri-docker.service

[root@k8s-master01 ~]# cat /usr/lib/systemd/system/cri-docker.socket

方法二：

wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.2.6/cri-dockerd-0.2.6-3.el7.x86_64.rpm

rpm -ivh cri-dockerd-0.2.6-3.el7.x86_64.rpm
systemctl enable cri-docker

重载沙箱

vim /usr/lib/systemd/system/cri-docker.service # 修改如下
ExecStart=/usr/bin/cri-dockerd --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8 --container-runtime-endpoint fd://

systemctl start cri-docker

查看可安装的kubernetes 的版本

yum list kubelet --showduplicates | sort -r
yum install -y kubelet-1.25.3 kubeadm-1.25.3 kubectl-1.25.3 选择安装最新版本

yum install kubeadm kubelet kubectl # 永远下载最新版

#安装后查看版本
$ kubeadm version

修改kubelet配置文件，为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容。

[root@k8s-master01 ~]# cat /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"

设置kubelet为开机自启动即可，由于没有生成配置文件，集群初始化后自动启动
systemctl enable kubelet
systemctl is-active kubelet

----
active
-----

生成配置脚本

kubeadm config print init-defaults > kubeadm-init.yaml

[root@k8s-node01 ~]# kubeadm config print init-defaults > kubeadm.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.30.119   #master 节点ip
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock   # 指定
imagePullPolicy: IfNotPresent
name: # master 节点主机名
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers    # 指定阿里云镜像地址
kind: ClusterConfiguration
kubernetesVersion: 1.25.2     # k8s版本
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16           # 新增字段指定pod网段
serviceSubnet: 10.96.0.0/12
scheduler: {}
---                           # 新增
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---                          # 新增
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

拉取需要的脚本

kubeadm config images pull --config=kubeadm-init.yaml

[root@k8s-master01 ~]# kubeadm config images pull --config=kubeadm-init.yaml
failed to pull image "registry.aliyuncs.com/google_containers/kube-apiserver:v1.25.3": output: time="2022-11-04T09:15:15+08:00" level=fatal msg="unable t
o determine image API version: rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing dial unix /var/run/containerd/containerd.sock: connect: no such file or directory\""

报错。解决方案：

注：安装docker或者containerd之后，默认在/etc/containerd/config.toml禁用了CRI，需要注释掉disabled_plugins = ["cri"]，否则执行kubeadm进行部署时会报错：

再重启docker containerd

[root@localhost ~]# systemctl restart docker containerd

再次执行故障解决

可能用到的命令

kubeadm init --config kubeadm-init.yaml | tee kubeadm-init.log

kubeadm reset --cri-socket=unix:///var/run/cri-dockerd.sock

执行下列命令

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=/etc/kubernetes/admin.conf

在node节点上执行下列命令

kubeadm join 192.168.30.119:6443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:8c0e860b3376eb871f8d4ec9f8cd16d8398ec1e2b65fce0cd629dfcf7c6cf3fd --cri-socket /var/run/cri-dockerd.sock # 注意带上参数--cri-socket

查看节点和pod，节点处于NotReady，coredns处于pending，主要是没有安装网络插件导致。

[root@k8s-master01 ~]# kubectl get nodes
NAME           STATUS     ROLES           AGE     VERSION
k8s-master01   NotReady   control-plane   8m15s   v1.25.3
k8s-node01     NotReady   <none>          2m47s   v1.25.3
[root@k8s-master01 ~]#
[root@k8s-master01 ~]# kubectl get pods -A
NAMESPACE     NAME                                   READY   STATUS    RESTARTS   AGE
kube-system   coredns-c676cc86f-86rp6                0/1     Pending   0          8m7s
kube-system   coredns-c676cc86f-x76q2                0/1     Pending   0          8m7s

下载网络插件

wget https://docs.projectcalico.org/manifests/calico.yaml --no-check-certificate

做出如下修改

# The default IPv4 pool to create on startup if none exists. Pod IPs will be
            # chosen from this range. Changing this value after installation will have
            # no effect. This should fall within `--cluster-cidr`.
            - name: CALICO_IPV4POOL_CIDR
              value: "10.244.0.0/16"
            # Disable file logging so `kubectl logs` works.
            - name: CALICO_DISABLE_FILE_LOGGING