k8s cridocker

mkdir /data/rpm
mount -t nfs 192.168.14.134:/data/rpm /data/rpm -o nolock,nfsvers=3,vers=3
echo "mount -t nfs 192.168.14.134:/data/rpm /data/rpm -o nolock,nfsvers=3,vers=3" >>/etc/rc.local
chmod +x /etc/rc.local 

cat >/root/base.repo<< EOF
[base]
name=base
baseurl=file:///data/rpm
enabled=1
gpgcheck=0
EOF 

rm -rf /etc/yum.repos.d/*
cp -r /root/base.repo /etc/yum.repos.d/ 

yum clean all 
yum makecache fast


cat > /etc/hosts <<EOF

192.168.14.132 master
192.168.14.133 node1
192.168.14.134 node2

192.168.14.135 node3


modprobe br_netfilter

echo "modprobe br_netfilter" >> /etc/profile

cat > /etc/sysctl.d/k8s.conf <EOF

net.bridge.bridge-nf-call-ip6tables = 1 

net.bridge.bridge-nf-call-iptables = 1 

net.ipv4.ip_forward = 1 

EOF

sysctl -p /etc/sysctl.d/k8s.conf


sudo sysctl --system


yum install -y ipset ipvsadm
cat > /etc/modules-load.d/ipvs.conf <<EOF
# Load IPVS at boot
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack_ipv4
EOF

systemctl enable --now systemd-modules-load.service

lsmod | grep -e ip_vs -e nf_conntrack_ipv4

yum install -y yum-utils device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel python-devel epel-release openssh-server socat ipvsadm conntrack ntpdate telnet

sudo yum install -y yum-utils

sudo yum install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y

sudo systemctl start docker

systemctl enable docker 

systemctl status docker

tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://19b12x6i.mirror.aliyuncs.com"],

"exec-opts": ["native.cgroupdriver=systemd"]

}
EOF
systemctl daemon-reload
systemctl enable docker
systemctl restart docker
systemctl status docker



tar -zxvf go1.20.2.linux-amd64.tar.gz -C /usr/local/


cat >> /etc/profile  << EOF
#go 环境变量
export GO111MODULE=on
export GOROOT=/usr/local/go
export GOPATH=/home/gopath
export PATH="$PATH:/usr/local/go/bin:/usr/local/go/bin"
EOF

source /etc/profile
go version

tar -zxvf cri.tar.gz 
cd cri-dockerd
mkdir bin
go build -o bin/cri-dockerd
mkdir -p /usr/local/bin
install -o root -g root -m 0755 bin/cri-dockerd /usr/local/bin/cri-dockerd
cp -a packaging/systemd/* /etc/systemd/system
sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
systemctl daemon-reload
systemctl enable cri-docker.service
systemctl enable --now cri-docker.socket
systemctl restart cri-docker.service
systemctl status cri-docker.service


ps -ef|grep dockerd

sed -i "/.*ExecStart=*/c\ExecStart=/usr/local/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint fd:// " /etc/systemd/system/cri-docker.service

systemctl daemon-reload
systemctl enable cri-docker.service
systemctl enable --now cri-docker.socket
systemctl restart cri-docker.service
systemctl status cri-docker.service
ps -ef|grep dockerd


yum install -y kubelet-1.26.2-0  kubeadm-1.26.2-0  kubectl-1.26.2-0 

cat <<EOF > /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF

systemctl enable kubelet && systemctl start kubelet


kubeadm init --kubernetes-version=1.26.2  \
 --apiserver-advertise-address=192.168.14.132 \
 --image-repository registry.aliyuncs.com/google_containers \
 --pod-network-cidr=10.244.0.0/16 \
 --cri-socket=unix:///var/run/cri-dockerd.sock
 
 
cat > /root/import.sh << EOF
 
#!/bin/bash


list_img=`ls /data/img`

cd /data/img 
for tmp in $list_img;do
    docker load -i $tmp 
done

EOF


  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config



 export KUBECONFIG=/etc/kubernetes/admin.conf
 kubeadm join 192.168.14.132:6443 --token wah770.26riwq1ujqvvcjuh \
    --discovery-token-ca-cert-hash sha256:28e0324bcc7a7b36aac7885b41595494f37a3d962951e2d234cc0201f37685fa 




[root@master ~]# kubectl get pods -n kube-system -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP               NODE     NOMINATED NODE   READINESS GATES
coredns-5bbd96d687-r7wng         0/1     Pending   0          44m   <none>           <none>   <none>           <none>
coredns-5bbd96d687-w7w5d         0/1     Pending   0          44m   <none>           <none>   <none>           <none>
etcd-master                      1/1     Running   0          44m   192.168.14.132   master   <none>           <none>
kube-apiserver-master            1/1     Running   0          44m   192.168.14.132   master   <none>           <none>
kube-controller-manager-master   1/1     Running   0          44m   192.168.14.132   master   <none>           <none>
kube-proxy-9pq26                 1/1     Running   0          44m   192.168.14.132   master   <none>           <none>
kube-scheduler-master            1/1     Running   0          44m   192.168.14.132   master   <none>           <none>



kubectl describe  pod coredns-5bbd96d687-lnsgj  -n kube-system 


Events:
  Type     Reason            Age                 From               Message
  ----     ------            ----                ----               -------
  Warning  FailedScheduling  62s (x10 over 46m)  default-scheduler  0/1 nodes are available: 1 node(s) had untolerated taint {node.kubernetes.io/not-ready: }. preemption: 0/1 nodes are available: 1 Preemption is not helpful for scheduling..
  

docker images|grep dns
registry.aliyuncs.com/google_containers/coredns                   v1.9.3    5185b96f0bec   9 months ago    48.8MB






kubeadm init --kubernetes-version=1.26.2  \
 --apiserver-advertise-address=192.168.14.132 \
 --image-repository registry.aliyuncs.com/google_containers \
 --pod-network-cidr=192.168.0.0/16   \
 --cri-socket=unix:///var/run/cri-dockerd.sock
 
 
kubeadm join 192.168.14.132:6443 --token 6q7ueo.1jt6zquc7hcr7ual \
    --discovery-token-ca-cert-hash sha256:46d4b6cf1c9295d97fdeffa60951f5d6f4f24c5835e7aa355e91ff2a241d9ca2
    



安装calico
kubectl apply -f tigera-operator.yaml
kubectl apply -f custom-resources.yaml


[root@master calico]# kubectl get pods  --all-namespaces
NAMESPACE          NAME                                       READY   STATUS    RESTARTS   AGE
calico-apiserver   calico-apiserver-5d6c8bdfb5-kgshv          1/1     Running   0          2m28s
calico-apiserver   calico-apiserver-5d6c8bdfb5-mvtjj          1/1     Running   0          2m28s
calico-system      calico-kube-controllers-6b7b9c649d-ls7s5   1/1     Running   0          2m42s
calico-system      calico-node-2z4lw                          1/1     Running   0          2m42s
calico-system      calico-typha-85b77679db-cpk9b              1/1     Running   0          2m42s
calico-system      csi-node-driver-jlmsh                      2/2     Running   0          2m39s
kube-system        coredns-5bbd96d687-lnsgj                   1/1     Running   0          12m
kube-system        coredns-5bbd96d687-rqrq5                   1/1     Running   0          12m
kube-system        etcd-master                                1/1     Running   0          13m
kube-system        kube-apiserver-master                      1/1     Running   0          13m
kube-system        kube-controller-manager-master             1/1     Running   0          13m
kube-system        kube-proxy-xwwpn                           1/1     Running   0          12m
kube-system        kube-scheduler-master                      1/1     Running   0          13m
tigera-operator    tigera-operator-54b47459dd-bd6mk           1/1     Running   0          3m26s


安装dashboard 

recommended.yaml
nodeName: master


kubectl get svc -n kubernetes-dashboard
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE
dashboard-metrics-scraper   ClusterIP   10.106.166.198   <none>        8000/TCP   2m35s
kubernetes-dashboard        ClusterIP   10.97.189.31     <none>        443/TCP    2m35s


kubectl edit svc kubernetes-dashboard  -n kubernetes-dashboard
type: NodePort

kubectl get svc -n kubernetes-dashboard
NAME                        TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)         AGE
dashboard-metrics-scraper   ClusterIP   10.106.166.198   <none>        8000/TCP        4m13s
kubernetes-dashboard        NodePort    10.97.189.31     <none>        443:31468/TCP   4m13s

https://192.168.14.132:31468

curl -I https://192.168.14.132:31468 -k
kubectl create clusterrolebinding dashboard-cluster-admin --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:kubernetes-dashboard


kubectl get secret -n kubernetes-dashboard

kubectl -n kubernetes-dashboard create token kubernetes-dashboard

eyJhbGciOiJSUzI1NiIsImtpZCI6ImpUV0ZXSDNodXEwaEVlZER0VDBvamVxSmlVdDY0QlNweDNBR1paNDZvNjgifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjc5NTYzOTY1LCJpYXQiOjE2Nzk1NjAzNjUsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInVpZCI6IjVmNWJjMWEyLTBmOWMtNGZiMi05ZTRhLThlNmJkMjQ2YjYwMSJ9fSwibmJmIjoxNjc5NTYwMzY1LCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQifQ.XDWobWH-E_aQsyUkLk25dU7Zzs1pSAXm-jk3QfyWA9b8AI-za3jGCy40GhdnekpT6gJUYfe9c_9FkTHGHJe20iXWwrIsrxTSiXKXRltJu6fSZQhc54vt6Iwmz1DSExmTAe9OPb4MIcFDW5-R9itQQrhL_zst6jMVrRoXuXDVIzAycJPeavqtPgP55ZBzHKX1w_0EhY6SIaYfO26Ss6gSBuLcBVrJSpVK6YwFNzQFldGOhDKqUWTlfkoxFOrhIOtVeE22KzJTV2GxQRAKb2vx06jwuWE1kR03j-YxadDGNOxtIklmVdGgVO8ZNLS6xaMNWF6Km1CpsBbs6ExMn3axVg


 kubectl get pods  --all-namespaces -o wide
NAMESPACE              NAME                                         READY   STATUS    RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
calico-apiserver       calico-apiserver-5d6c8bdfb5-kgshv            1/1     Running   0          14m     192.168.219.70   master   <none>           <none>
calico-apiserver       calico-apiserver-5d6c8bdfb5-mvtjj            1/1     Running   0          14m     192.168.219.69   master   <none>           <none>
calico-system          calico-kube-controllers-6b7b9c649d-ls7s5     1/1     Running   0          14m     192.168.219.66   master   <none>           <none>
calico-system          calico-node-2z4lw                            1/1     Running   0          14m     192.168.14.132   master   <none>           <none>
calico-system          calico-typha-85b77679db-cpk9b                1/1     Running   0          14m     192.168.14.132   master   <none>           <none>
calico-system          csi-node-driver-jlmsh                        2/2     Running   0          14m     192.168.219.65   master   <none>           <none>
kube-system            coredns-5bbd96d687-lnsgj                     1/1     Running   0          24m     192.168.219.67   master   <none>           <none>
kube-system            coredns-5bbd96d687-rqrq5                     1/1     Running   0          24m     192.168.219.68   master   <none>           <none>
kube-system            etcd-master                                  1/1     Running   0          24m     192.168.14.132   master   <none>           <none>
kube-system            kube-apiserver-master                        1/1     Running   0          25m     192.168.14.132   master   <none>           <none>
kube-system            kube-controller-manager-master               1/1     Running   0          25m     192.168.14.132   master   <none>           <none>
kube-system            kube-proxy-xwwpn                             1/1     Running   0          24m     192.168.14.132   master   <none>           <none>
kube-system            kube-scheduler-master                        1/1     Running   0          25m     192.168.14.132   master   <none>           <none>
kubernetes-dashboard   dashboard-metrics-scraper-5cc596bbc7-g244g   1/1     Running   0          7m11s   192.168.219.72   master   <none>           <none>
kubernetes-dashboard   kubernetes-dashboard-cf4b945fc-xmwpk         1/1     Running   0          7m11s   192.168.219.71   master   <none>           <none>
tigera-operator        tigera-operator-54b47459dd-bd6mk             1/1     Running   0          15m     192.168.14.132   master   <none>           <none>



    
节点注册

cat <<EOF > /etc/sysconfig/kubelet
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF


sudo kubeadm reset --cri-socket /var/run/cri-dockerd.sock

sudo systemctl enable docker

sudo systemctl enable kubelet

sudo systemctl daemon-reload

sudo systemctl restart docker
systemctl restart cri-docker.service

sudo netstat -lnp | grep 1025

sudo rm -rf /etc/kubernetes/kubelet.conf /etc/kubernetes/pki/ca.crt
pkill -9 kubelet



kubeadm join 192.168.14.132:6443 --token xu6hfw.jfykuy0st3k3r69u --discovery-token-ca-cert-hash sha256:46d4b6cf1c9295d97fdeffa60951f5d6f4f24c5835e7aa355e91ff2a241d9ca2  --cri-socket /var/run/cri-dockerd.sock



[root@master dashboard]#  kubectl get pods  --all-namespaces -o wide
NAMESPACE              NAME                                         READY   STATUS    RESTARTS   AGE     IP                NODE     NOMINATED NODE   READINESS GATES
calico-apiserver       calico-apiserver-5d6c8bdfb5-kgshv            1/1     Running   0          40m     192.168.219.70    master   <none>           <none>
calico-apiserver       calico-apiserver-5d6c8bdfb5-mvtjj            1/1     Running   0          40m     192.168.219.69    master   <none>           <none>
calico-system          calico-kube-controllers-6b7b9c649d-ls7s5     1/1     Running   0          40m     192.168.219.66    master   <none>           <none>
calico-system          calico-node-2z4lw                            1/1     Running   0          40m     192.168.14.132    master   <none>           <none>
calico-system          calico-node-586xg                            1/1     Running   0          8m44s   192.168.14.135    node3    <none>           <none>
calico-system          calico-node-n5z9h                            1/1     Running   0          2m17s   192.168.14.133    node1    <none>           <none>
calico-system          calico-typha-85b77679db-cpk9b                1/1     Running   0          40m     192.168.14.132    master   <none>           <none>
calico-system          calico-typha-85b77679db-wft27                1/1     Running   0          2m9s    192.168.14.135    node3    <none>           <none>
calico-system          csi-node-driver-hjj4j                        2/2     Running   0          2m12s   192.168.166.129   node1    <none>           <none>
calico-system          csi-node-driver-jlmsh                        2/2     Running   0          40m     192.168.219.65    master   <none>           <none>
calico-system          csi-node-driver-xgr8s                        2/2     Running   0          8m43s   192.168.135.1     node3    <none>           <none>
kube-system            coredns-5bbd96d687-lnsgj                     1/1     Running   0          51m     192.168.219.67    master   <none>           <none>
kube-system            coredns-5bbd96d687-rqrq5                     1/1     Running   0          51m     192.168.219.68    master   <none>           <none>
kube-system            etcd-master                                  1/1     Running   0          51m     192.168.14.132    master   <none>           <none>
kube-system            kube-apiserver-master                        1/1     Running   0          51m     192.168.14.132    master   <none>           <none>
kube-system            kube-controller-manager-master               1/1     Running   0          51m     192.168.14.132    master   <none>           <none>
kube-system            kube-proxy-8n6vw                             1/1     Running   0          8m44s   192.168.14.135    node3    <none>           <none>
kube-system            kube-proxy-p7758                             1/1     Running   0          2m17s   192.168.14.133    node1    <none>           <none>
kube-system            kube-proxy-xwwpn                             1/1     Running   0          51m     192.168.14.132    master   <none>           <none>
kube-system            kube-scheduler-master                        1/1     Running   0          51m     192.168.14.132    master   <none>           <none>
kubernetes-dashboard   dashboard-metrics-scraper-5cc596bbc7-g244g   1/1     Running   0          33m     192.168.219.72    master   <none>           <none>
kubernetes-dashboard   kubernetes-dashboard-cf4b945fc-xmwpk         1/1     Running   0          33m     192.168.219.71    master   <none>           <none>
tigera-operator        tigera-operator-54b47459dd-bd6mk             1/1     Running   0          41m     192.168.14.132    master   <none>           <none>