一、环境配置
服务器配置：2核4G
IP：192.168.10.23
主机名：master4
将改主机加入此 集群
# 1.修改主机名
hostnamectl set-hostname master4 && bash
# 2.添加hosts
127.0.1.1 master4
192.168.10.20 master
192.168.10.21 master2
192.168.10.22 master3
192.168.10.23 master4
192.168.10.24 node1
192.168.10.25 node2
# 3.ssh信任
ssh-keygen -t rsa
ssh-copy-id master
# 4.关闭交换分区
swapoff -a # 临时关闭
永久关闭为注销/etc/fstab中swap一行
# 5.修改机器内核参数
modprobe br_netfilter
echo "modprobe br_netfilter" >> /etc/profile
cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
sysctl -p /etc/sysctl.d/k8s.conf
# 6. 关闭防火墙
systemctl stop firewalld ; systemctl disable firewalld
# 7.关闭selinux,修改 x selinux 配置文件之后，重启：ubuntu如果没有安装不用管
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
# 8.配置阿里云源
# step 1: 安装必要的一些系统工具
sudo apt-get update
sudo apt-get -y install apt-transport-https ca-certificates curl software-properties-common
# step 2: 安装GPG证书
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
# Step 3: 写入软件源信息
sudo add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
# Step 4: 更新并安装Docker-CE
sudo apt-get -y update
# 9.配置kubernets源
# (1).更新 apt 包索引并安装使用 Kubernetes apt 仓库所需要的包
sudo apt-get update
# apt-transport-https 可能是一个虚拟包（dummy package）；如果是的话，你可以跳过安装这个包
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
# (2).下载用于 Kubernetes 软件包仓库的公共签名密钥。所有仓库都使用相同的签名密钥，因此你可以忽略URL中的版本：
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
# (3).添加 Kubernetes apt 仓库。 请注意，此仓库仅包含适用于 Kubernetes 1.28 的软件包； 对于其他 Kubernetes 次要版本，则需要更改 URL 中的 Kubernetes 次要版本以匹配你所需的次要版本，如果之前有会覆盖
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
# 10.时间同步并定时同步
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
ntpdate time1.aliyun.com
二、基础软件包安装
sudo apt -y install ipvsadm ipset sysstat conntrack g++ openssl libssl-dev zlib zlib1g-dev libpcre3 libpcre3-dev build-essential
2.1 安装containerd
# 1.安装containerd服务
yum -y install containerd
# 2.生成containerd配置文件
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
# 3.修改配置文件
vim /etc/containerd/config.toml
SystemdCgroup = true # false改为true
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9" # 如果版本不清楚后面kubeadm config images list --config=kubeadm.yml时可以看了再修改
# 4.配置为开机启动
systemctl enable containerd --now
# 5.修改/etc/crictl.yaml 文件
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
systemctl restart containerd
# 6.配置镜像加速器
# 编辑 vim /etc/containerd/config.toml 文件，修改
config_path = "/etc/containerd/certs.d"
mkdir /etc/containerd/certs.d/docker.io/ -p
vim /etc/containerd/certs.d/docker.io/hosts.toml
[host."https://pft7f97f.mirror.aliyuncs.com",host."https://registry.docker-cn.com",host."https://docker.mirrors.ustc.edu.cn"]
capabilities = ["pull"]
systemctl restart containerd
# 设置容器运行时
# 1.设置容器运行时，master,node
crictl config runtime-endpoint unix:///run/containerd/containerd.sock
2.2 安装k8s所需软件
# 安装 kubelet、kubeadm 和 kubectl，并锁定其版本：
sudo apt-get update -y
# 1.查看kubeadm、kubectl、kubelet有哪些版本
root@master4:/etc/containerd# apt-cache madison kubeadm kubelet kubectl
kubeadm | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubeadm | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubeadm | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubeadm | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubeadm | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubeadm | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubelet | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
kubectl | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb Packages
# 2.安装指定版本
root@master4:/etc/containerd# apt-get -y install kubelet=1.28.2-1.1 kubeadm=1.28.2-1.1 kubectl=1.28.2-1.1
# 阻止软件自动更新
apt-mark hold kubelet kubeadm kubectl
systemctl enable kubelet
2.3 安装配置nginx+keepalived
# 安装nginx、keepalived
apt-get install nginx keepalived
# 配置文件
[root@master service]# scp /etc/keepalived/keepalived.conf master4:/etc/keepalived/
keepalived.conf 100% 806 85.4KB/s 00:00
[root@master service]# scp /etc/keepalived/check_nginx.sh master4:/etc/keepalived/
check_nginx.sh
[root@master service]# scp /etc/nginx/nginx.conf master4:/etc/nginx/
nginx.conf
# 在master4的nginx.conf中添加一条启用stream模块
include /etc/nginx/modules-enabled/*.conf;
systemctl enable nginx && systemctl start nginx
systemctl enable keepalived && systemctl start keepalived
2.4 加入到集群中作为控制节点
# 2.将master节点证书拷贝到master4节点，master4节点建立目录
mkdir -p /etc/kubernetes/pki/etcd/
scp /etc/kubernetes/pki/ca.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.* master4:/etc/kubernetes/pki/etcd/
# 将kubeadm.yaml文件传输到master4上下载kubernetes所需镜像
root@master4:~# kubeadm config images list --config=kubeadm.yaml
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9
registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.9-0
registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.10.1
root@master4:~# kubeadm config images pull--config=kubeadm.yaml
# 在master节点生成token
[root@master k8s]# kubeadm token create --print-join-command
kubeadm join 192.168.10.19:16443 --token 03b1fu.tyfrsq69b28jcduj --discovery-token-ca-cert-hash sha256:3d2052ebcdc58cce07aeb55f9e5987d8d406e3b0d0370299283cdb4fdc216eeb
# 将master4加入集群中
kubeadm join 192.168.10.19:16443 --token 03b1fu.tyfrsq69b28jcduj --discovery-token-ca-cert-hash sha256:3d2052ebcdc58cce07aeb55f9e5987d8d406e3b0d0370299283cdb4fdc216eeb --control-plane
# master4上执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 查看nodes
root@master4:/etc/nginx# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master Ready control-plane 2d11h v1.28.5
master2 Ready control-plane 2d11h v1.28.5
master4 Ready control-plane 4m3s v1.28.2
node1 Ready worker 2d10h v1.28.5
node2 Ready worker 30h v1.28.5
三、版本升级：从1.28.2升级到1.28.5
# 1.将master4设置为维护模式
kubectl cordon master4
# 2.驱逐节点上数据
kubectl drain master4 --ignore-daemonsets --force --delete-empty-data
# 3.查看可升级的版本
apt update
apt-cache madison kubeadm
# 4.升级kubeadm,用最新的补丁版本号替换 1.28.x-* 中的 x
apt-mark unhold kubeadm && \
apt-get update && apt-get install -y kubeadm='1.28.x-*' && \
apt-mark hold kubeadm
# 查看版本号
kubectl version
# 5.验证版本升级计划
root@master4:/etc/apt/keyrings# kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.28.5
[upgrade/versions] kubeadm version: v1.28.5
I0113 11:32:45.140007 382936 version.go:256] remote version is much newer: v1.29.0; falling back to: stable-1.28
[upgrade/versions] Target version: v1.28.5
[upgrade/versions] Latest version in the v1.28 series: v1.28.5
# 6.选择要升级到的目标版本，运行合适的命令
root@master4:/etc/apt/keyrings# kubeadm upgrade apply v1.28.5
成功命令：
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!
root@master4:/etc/apt/keyrings# kubeadm upgrade apply v1.28.5
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade/version] You have chosen to change the cluster version to "v1.28.5"
[upgrade/versions] Cluster version: v1.28.5
[upgrade/versions] kubeadm version: v1.28.5
[upgrade] Are you sure you want to proceed? [y/N]: y
[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster
[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection
[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'
W0113 11:41:29.087629 387705 checks.go:835] detected that the sandbox image "registry.aliyuncs.com/google_containers/pause:3.7" of the container runtime is inconsistent with that used by kubeadm. It is recommended that using "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9" as the CRI sandbox image.
[upgrade/apply] Upgrading your Static Pod-hosted control plane to version "v1.28.5" (timeout: 5m0s)...
[upgrade/etcd] Upgrading to TLS for etcd
[upgrade/staticpods] Preparing for "etcd" upgrade
[upgrade/staticpods] Renewing etcd-server certificate
[upgrade/staticpods] Renewing etcd-peer certificate
[upgrade/staticpods] Renewing etcd-healthcheck-client certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/etcd.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2024-01-13-11-41-52/etcd.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
[apiclient] Found 3 Pods for label selector component=etcd
[upgrade/staticpods] Component "etcd" upgraded successfully!
[upgrade/etcd] Waiting for etcd to become available
[upgrade/staticpods] Writing new Static Pod manifests to "/etc/kubernetes/tmp/kubeadm-upgraded-manifests3806239562"
[upgrade/staticpods] Preparing for "kube-apiserver" upgrade
[upgrade/staticpods] Current and new manifests of kube-apiserver are equal, skipping upgrade
[upgrade/staticpods] Preparing for "kube-controller-manager" upgrade
[upgrade/staticpods] Current and new manifests of kube-controller-manager are equal, skipping upgrade
[upgrade/staticpods] Preparing for "kube-scheduler" upgrade
[upgrade/staticpods] Current and new manifests of kube-scheduler are equal, skipping upgrade
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upgrade] Backing up kubelet config file to /etc/kubernetes/tmp/kubeadm-kubelet-config188327240/config.yaml
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[addons] Applied essential addon: CoreDNS
W0113 11:44:01.589812 387705 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!
[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.
kubeadm upgrade apply v1.28.5# 7.升级 kubelet 和 kubectl
apt-mark unhold kubeadm kubectl
apt-get update && apt-get install -y kubelet='1.28.5-*' kubectl='1.28.5-*'
验证版本：
kubectl version
kubelet --version
apt-mark hold kubelet kubectl
# 7.重启服务
sudo systemctl daemon-reload
sudo systemctl restart kubelet
# 8.解除保护，将节点加入集群
root@master4:/etc/apt/keyrings# kubectl uncordon master4
node/master4 uncordoned
root@master4:/etc/apt/keyrings# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master Ready control-plane 2d22h v1.28.5
master2 Ready control-plane 2d22h v1.28.5
master4 Ready control-plane 10h v1.28.5
node1 Ready worker 2d20h v1.28.5
node2 Ready worker 41h v1.28.5
