基于kubeadm和containerd部署单master k8s v1.24.2
一、加载内核参数优化
cat > /etc/sysctl.d/k8s.conf <<EOF net.bridge.bridge-nf-call-iptables=1 net.bridge.bridge-nf-call-ip6tables=1 net.bridge.bridge-nf-call-arptables = 1 net.ipv4.ip_forward=1 net.ipv4.tcp_tw_recycle=0 net.core.somaxconn = 32768 vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它 vm.overcommit_memory=1 # 不检查物理内存是否够用 vm.panic_on_oom=0 # 开启 OOM fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 fs.file-max=52706963 fs.nr_open=52706963 net.ipv6.conf.all.disable_ipv6=1 net.netfilter.nf_conntrack_max=2310720 net.ipv4.conf.all.rp_filter = 1 net.ipv4.neigh.default.gc_thresh1 = 80000 net.ipv4.neigh.default.gc_thresh2 = 90000 net.ipv4.neigh.default.gc_thresh3 = 100000 EOF sysctl -p /etc/sysctl.d/k8s.conf modprobe br_netfilter cat > /etc/sysconfig/modules/ipvs.modules <<EOF #!/bin/bash modprobe -- ip_vs modprobe -- ip_vs_rr modprobe -- ip_vs_wrr modprobe -- ip_vs_sh modprobe -- nf_conntrack modprobe br_netfilter EOF
二、配置相关仓库源
cat<<END> install-1.sh #!/bin/bash #yum源配置 systemctl stop firewalld.service && systemctl disable firewalld.service yum install ipset vim ipvsadm wget bash-completion.noarch -y yum install -y yum-utils device-mapper-persistent-data lvm2 yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo cat <<EOF > /etc/yum.repos.d/kubernetes.repo [kubernetes] name=Kubernetes baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg EOF cat <<EOF >> /etc/security/limits.conf root soft nofile 1048576 root hard nofile 1048576 root soft stack 10240 EOF # 关闭selinux setenforce 0 sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config grep --color=auto '^SELINUX' /etc/selinux/config #关闭Swap swapoff -a sed -i 's/.*swap.*/#&/' /etc/fstab yum clean all && yum makecache -y yum install -y kubelet-1.24.2 kubeadm-1.24.2 kubectl-1.24.2 && yum install -y containerd.io systemctl enable kubelet containerd && systemctl start kubelet modprobe br_netfilter sysctl --system sysctl -p /etc/sysctl.d/k8s.conf chmod +x /etc/sysconfig/modules/ipvs.modules && /etc/sysconfig/modules/ipvs.modules lsmod | grep -e ip_vs -e nf_conntrack_ipv4 cut -f1 -d " " /proc/modules | grep -e ip_vs -e nf_conntrack_ipv4 lsmod | grep ip_vs END chmod +x ./install-1.sh ./install-1.sh
三、containerd配置
cat<<END> install-2.sh #!/bin/bash containerd config default > /etc/containerd/config.toml sed -i "s#k8s.gcr.io/pause:3.6#registry.aliyuncs.com/google_containers/pause:3.6#g" /etc/containerd/config.toml sed -i "s#https://registry-1.docker.io#https://0k0953tv.mirror.aliyuncs.com#g" /etc/containerd/config.toml sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml systemctl restart containerd systemctl status containerd.service #wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.24.2/crictl-v1.24.2-linux-amd64.tar.gz 推荐提前下载 tar zxvf crictl-v1.24.2-linux-amd64.tar.gz -C /usr/local/bin cat > /etc/crictl.yaml <<EOF runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF crictl config runtime-endpoint unix:/run/containerd/containerd.sock echo "测试拉取镜像" crictl pull nginx crictl images END chmod +x ./install-2.sh ./install-2.sh
四、初始化k8s集群
[root@containerd-k8s-01 ~]# kubeadm init --image-repository registry.aliyuncs.com/google_containers --pod-network-cidr=192.168.0.0/16 [init] Using Kubernetes version: v1.24.3 [preflight] Running pre-flight checks [WARNING FileExisting-tc]: tc not found in system path [preflight] Pulling images required for setting up a Kubernetes cluster [preflight] This might take a minute or two, depending on the speed of your internet connection [preflight] You can also perform this action in beforehand using 'kubeadm config images pull' [certs] Using certificateDir folder "/etc/kubernetes/pki" [certs] Generating "ca" certificate and key [certs] Generating "apiserver" certificate and key [certs] apiserver serving cert is signed for DNS names [containerd-k8s-01 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.247.111] [certs] Generating "apiserver-kubelet-client" certificate and key [certs] Generating "front-proxy-ca" certificate and key [certs] Generating "front-proxy-client" certificate and key [certs] Generating "etcd/ca" certificate and key [certs] Generating "etcd/server" certificate and key [certs] etcd/server serving cert is signed for DNS names [containerd-k8s-01 localhost] and IPs [192.168.247.111 127.0.0.1 ::1] [certs] Generating "etcd/peer" certificate and key [certs] etcd/peer serving cert is signed for DNS names [containerd-k8s-01 localhost] and IPs [192.168.247.111 127.0.0.1 ::1] [certs] Generating "etcd/healthcheck-client" certificate and key [certs] Generating "apiserver-etcd-client" certificate and key [certs] Generating "sa" key and public key [kubeconfig] Using kubeconfig folder "/etc/kubernetes" [kubeconfig] Writing "admin.conf" kubeconfig file [kubeconfig] Writing "kubelet.conf" kubeconfig file [kubeconfig] Writing "controller-manager.conf" kubeconfig file [kubeconfig] Writing "scheduler.conf" kubeconfig file [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" [kubelet-start] Starting the kubelet [control-plane] Using manifest folder "/etc/kubernetes/manifests" [control-plane] Creating static Pod manifest for "kube-apiserver" [control-plane] Creating static Pod manifest for "kube-controller-manager" [control-plane] Creating static Pod manifest for "kube-scheduler" [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests" [wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s [apiclient] All control plane components are healthy after 32.507631 seconds [upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace [kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster [upload-certs] Skipping phase. Please see --upload-certs [mark-control-plane] Marking the node containerd-k8s-01 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers] [mark-control-plane] Marking the node containerd-k8s-01 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule node-role.kubernetes.io/control-plane:NoSchedule] [bootstrap-token] Using token: mi917i.3krys8xwff30pkgq [bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles [bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes [bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials [bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token [bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster [bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key [addons] Applied essential addon: CoreDNS [addons] Applied essential addon: kube-proxy Your Kubernetes control-plane has initialized successfully! To start using your cluster, you need to run the following as a regular user: mkdir -p $HOME/.kube sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config sudo chown $(id -u):$(id -g) $HOME/.kube/config Alternatively, if you are the root user, you can run: export KUBECONFIG=/etc/kubernetes/admin.conf You should now deploy a pod network to the cluster. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at: https://kubernetes.io/docs/concepts/cluster-administration/addons/ Then you can join any number of worker nodes by running the following on each as root: kubeadm join 192.168.247.111:6443 --token mi917i.3krys8xwff30pkgq \ --discovery-token-ca-cert-hash sha256:cce6131724253b7e3b157557581f6bffe708d176331fe2e58bb5677959ad346c [root@containerd-k8s-01 ~]# mkdir -p $HOME/.kube [root@containerd-k8s-01 ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config [root@containerd-k8s-01 ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config [root@containerd-k8s-01 ~]# export KUBECONFIG=/etc/kubernetes/admin.conf [root@containerd-k8s-01 ~]# [root@containerd-k8s-01 ~]# crictl images IMAGE TAG IMAGE ID SIZE docker.io/library/nginx latest 41b0e86104ba6 56.7MB registry.aliyuncs.com/google_containers/coredns v1.8.6 a4ca41631cc7a 13.6MB registry.aliyuncs.com/google_containers/etcd 3.5.3-0 aebe758cef4cd 102MB registry.aliyuncs.com/google_containers/kube-apiserver v1.24.3 d521dd763e2e3 33.8MB registry.aliyuncs.com/google_containers/kube-controller-manager v1.24.3 586c112956dfc 31MB registry.aliyuncs.com/google_containers/kube-proxy v1.24.3 2ae1ba6417cbc 39.5MB registry.aliyuncs.com/google_containers/kube-scheduler v1.24.3 3a5aa3a515f5d 15.5MB registry.aliyuncs.com/google_containers/pause 3.6 6270bb605e12e 302kB registry.aliyuncs.com/google_containers/pause 3.7 221177c6082a8 311kB [root@containerd-k8s-01 ~]# [root@containerd-k8s-01 ~]# [root@containerd-k8s-02 ~]# kubeadm join 192.168.247.111:6443 --token mi917i.3krys8xwff30pkgq \ > --discovery-token-ca-cert-hash sha256:cce6131724253b7e3b157557581f6bffe708d176331fe2e58bb5677959ad346c [preflight] Running pre-flight checks [WARNING FileExisting-tc]: tc not found in system path [preflight] Reading configuration from the cluster... [preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml' [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" [kubelet-start] Starting the kubelet [kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap... This node has joined the cluster: * Certificate signing request was sent to apiserver and a response was received. * The Kubelet was informed of the new secure connection details. Run 'kubectl get nodes' on the control-plane to see this node join the cluster. [root@containerd-k8s-02 ~]# [root@containerd-k8s-03 ~]# kubeadm join 192.168.247.111:6443 --token mi917i.3krys8xwff30pkgq \ > --discovery-token-ca-cert-hash sha256:cce6131724253b7e3b157557581f6bffe708d176331fe2e58bb5677959ad346c [preflight] Running pre-flight checks [WARNING FileExisting-tc]: tc not found in system path [preflight] Reading configuration from the cluster... [preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml' [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml" [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env" [kubelet-start] Starting the kubelet [kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap... This node has joined the cluster: * Certificate signing request was sent to apiserver and a response was received. * The Kubelet was informed of the new secure connection details. Run 'kubectl get nodes' on the control-plane to see this node join the cluster. [root@containerd-k8s-03 ~]# [root@containerd-k8s-01 ~]# kubectl get node -owide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME containerd-k8s-01 NotReady control-plane 19m v1.24.2 192.168.247.111 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 containerd-k8s-02 NotReady <none> 65s v1.24.2 192.168.247.112 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 containerd-k8s-03 NotReady <none> 58s v1.24.2 192.168.247.113 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 [root@containerd-k8s-01 ~]# kubectl get pod -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-74586cf9b6-6r54q 0/1 Pending 0 19m kube-system coredns-74586cf9b6-prqx5 0/1 Pending 0 19m kube-system etcd-containerd-k8s-01 1/1 Running 0 19m kube-system kube-apiserver-containerd-k8s-01 1/1 Running 0 19m kube-system kube-controller-manager-containerd-k8s-01 1/1 Running 0 19m kube-system kube-proxy-5wsxl 1/1 Running 0 19m kube-system kube-proxy-ndvzb 1/1 Running 0 72s kube-system kube-proxy-zjdng 1/1 Running 0 65s kube-system kube-scheduler-containerd-k8s-01 1/1 Running 0 19m [root@containerd-k8s-01 ~]#
五、安装calico网络组件
[root@containerd-k8s-01 ~]# wget https://docs.projectcalico.org/manifests/calico.yaml --2022-07-19 10:04:32-- https://docs.projectcalico.org/manifests/calico.yaml Resolving docs.projectcalico.org (docs.projectcalico.org)... 52.220.193.16, 18.139.201.98, 2406:da18:880:3802:371c:4bf1:923b:fc30, ... Connecting to docs.projectcalico.org (docs.projectcalico.org)|52.220.193.16|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 229966 (225K) [text/yaml] Saving to: ‘calico.yaml’ calico.yaml 100%[===================================================================================================================>] 224.58K 602KB/s in 0.4s 2022-07-19 10:04:32 (602 KB/s) - ‘calico.yaml’ saved [229966/229966] [root@containerd-k8s-01 ~]# ls anaconda-ks.cfg calico.yaml crictl-v1.24.2-linux-amd64.tar.gz install-1.sh install-2.sh [root@containerd-k8s-01 ~]# kubectl apply -f calico.yaml configmap/calico-config created customresourcedefinition.apiextensions.k8s.io/bgpconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/bgppeers.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/blockaffinities.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/caliconodestatuses.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/clusterinformations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/felixconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworkpolicies.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/globalnetworksets.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/hostendpoints.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamblocks.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamconfigs.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipamhandles.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ippools.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/ipreservations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/kubecontrollersconfigurations.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networkpolicies.crd.projectcalico.org created customresourcedefinition.apiextensions.k8s.io/networksets.crd.projectcalico.org created clusterrole.rbac.authorization.k8s.io/calico-kube-controllers created clusterrolebinding.rbac.authorization.k8s.io/calico-kube-controllers created clusterrole.rbac.authorization.k8s.io/calico-node created clusterrolebinding.rbac.authorization.k8s.io/calico-node created daemonset.apps/calico-node created serviceaccount/calico-node created deployment.apps/calico-kube-controllers created serviceaccount/calico-kube-controllers created poddisruptionbudget.policy/calico-kube-controllers created [root@containerd-k8s-01 ~]# [root@containerd-k8s-01 ~]# kubectl get node -owide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME containerd-k8s-01 Ready control-plane 56m v1.24.2 192.168.247.111 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 containerd-k8s-02 Ready <none> 37m v1.24.2 192.168.247.112 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 containerd-k8s-03 Ready <none> 37m v1.24.2 192.168.247.113 <none> CentOS Linux 8 4.18.0-348.7.1.el8_5.x86_64 containerd://1.6.6 [root@containerd-k8s-01 ~]#
参考文档：https://blog.csdn.net/weixin_42562106/article/details/118972863
