k8s 1.24.1 containerd 证书过期处理

在 master 节点上进行操作

[root@k8s-master-1 ~]# kubectl get nodes
Unable to connect to the server: x509: certificate has expired or is not yet valid: current time 2023-10-23T11:06:58+08:00 is after 2023-09-14T10:26:34Z

# 检查证书的过期时间
kubeadm certs check-expiration

# 备份原来的证书
cp -r /etc/kubernetes  /etc/kubernetes_bak

# 重新生成证书
kubeadm certs renew all

# 再次查看证书
kubeadm  certs check-expiration

# 出现没有认证的情况
[root@k8s-master-1 ~]# kubectl get nodes
error: You must be logged in to the server (Unauthorized)

# 把新的配置覆盖
cp -rp $HOME/.kube/config $HOME/.kube/config.bak 
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config

[root@k8s-master-1 ~]# kubectl get nodes -o wide
NAME           STATUS   ROLES           AGE    VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION               CONTAINER-RUNTIME
k8s-master-1   Ready    control-plane   403d   v1.24.1   172.16.16.108   <none>        CentOS Linux 7 (Core)   5.19.8-1.el7.elrepo.x86_64   containerd://1.6.8
k8s-node-1     Ready    <none>          403d   v1.24.1   172.16.16.109   <none>        CentOS Linux 7 (Core)   5.19.8-1.el7.elrepo.x86_64   containerd://1.6.8
k8s-node-2     Ready    <none>          403d   v1.24.1   172.16.16.110   <none>        CentOS Linux 7 (Core)   5.19.8-1.el7.elrepo.x86_64   containerd://1.6.8

# 注意，这时 K8S 集群还是不可用，kubelet 的证书并没有更新成功
# 参考官方文档：https://v1-24.docs.kubernetes.io/zh-cn/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm/#kubelet-client-cert

# 重新生成 master 证书 
mkdir -p /data/kubelet_key/k8s-master-1
cd /data/kubelet_key/k8s-master-1

## 注意 k8s-master-1 名字要跟 kubect get nodes 的一致；--config=/root/kubeadm.yaml，该配置是 kubeadmin 初始化时所用的配置
kubeadm kubeconfig user --org system:nodes --client-name system:node:k8s-master-1 --config=/root/kubeadm.yaml > kubelet.conf
mkdir -p /data/k8s_bak/kubelet
mv /etc/kubernetes/kubelet.conf /data/k8s_bak/kubelet
cp /data/kubelet_key/k8s-master-1/kubelet.conf /etc/kubernetes/
mv /var/lib/kubelet/pki/kubelet-client-* /data/k8s_bak/kubelet/
systemctl restart kubelet
systemctl status kubelet

# 重启 kube-apiserver, kube-controller-manager, kube-scheduler and etcd
mkdir -p /data/k8s_bak/manifests
mv /etc/kubernetes/manifests/*.yaml /data/k8s_bak/manifests
## 过20秒后
mv /data/k8s_bak/manifests/*.yaml /etc/kubernetes/manifests/

# 等待 pod 重启完成
kubectl -n kube-system get pod 

# 生成 node1 证书 
mkdir -p /data/kubelet_key/k8s-node-1
cd /data/kubelet_key/k8s-node-1
kubeadm kubeconfig user --org system:nodes --client-name system:node:k8s-node-1 --config=/root/kubeadm.yaml > kubelet.conf

## 传输到 k8s-node-1
scp kubelet.conf k8s-node-1:/tmp/

# 生成 node2 证书
mkdir -p /data/kubelet_key/k8s-node-2
cd /data/kubelet_key/k8s-node-2
kubeadm kubeconfig user --org system:nodes --client-name system:node:k8s-node-2 --config=/root/kubeadm.yaml > kubelet.conf

## 传输到 k8s-node-2
scp kubelet.conf k8s-node-2:/tmp/

在 node1，node2 上重新配置 kubelet

## node1,node2 执行相同的操作
mkdir -p /data/k8s_bak/kubelet
mv /etc/kubernetes/kubelet.conf /data/k8s_bak/kubelet
mv /var/lib/kubelet/pki/kubelet-client-* /data/k8s_bak/kubelet/
cp /tmp/kubelet.conf /etc/kubernetes/
systemctl restart kubelet
systemctl status kubelet

测试

cd /tmp/
cat >> deployment.yaml << EOF 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    name: nginx
  name: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: nginx
        image: nginx 
        imagePullPolicy: IfNotPresent
EOF

kubectl apply -f deployment.yaml

kubectl get pod

参考文档

https://v1-24.docs.kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/kubeadm-certs/
https://v1-24.docs.kubernetes.io/zh-cn/docs/setup/production-environment/tools/kubeadm/troubleshooting-kubeadm/#kubelet-client-cert