二进制k8s 集群新增加node 节点

环境

名称	ip地址	cpu	内存
lgy-k8s-master0021	10.65.0.21	4c	8G
lgy-k8s-node0012	10.65.0.12	4c	8G

node节点初始化（以新增加 lgy-k8s-node0012 节点为例）

#!/bin/sh
# 安装yum源
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

# 安装
yum --enablerepo=elrepo-kernel install kernel-ml-devel kernel-ml -y

# 设置生成新的grub
grub2-set-default 0
grub2-mkconfig -o /etc/grub2.cfg


# 移除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64 -y

# 安装新版本
yum --disablerepo=* --enablerepo=elrepo-kernel install -y kernel-ml-tools.x86_64

# 重启
reboot

# 查看内核版本
uname -sr

#1.修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2
net.ipv4.ip_forward = 1
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_synack_retries = 2
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.netfilter.nf_conntrack_max = 2310720
fs.inotify.max_user_watches=89100
fs.may_detach_mounts = 1
fs.file-max = 52706963
fs.nr_open = 52706963
net.bridge.bridge-nf-call-arptables = 1
vm.swappiness = 0   #最大限度使用物理内存，然后才是 swap空间
vm.overcommit_memory=1
vm.panic_on_oom=0
EOF
sysctl --system

#2. 临时关闭
swapoff -a
#3. 永久关闭
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

#4. 开启ipvs
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
# 查看是否加载
lsmod | grep ip_vs
# 配置开机自加载
cat <<EOF>> /etc/rc.local
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod +x /etc/rc.d/rc.local
#5. 关闭sellinux
#临时关闭
setenforce 0
#永久关闭
sed -i 's#SELINUX=enforcing#SELINUX=Disabled#g'  /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=Disabled#g'  /etc/selinux/config
#6.禁用postfix
systemctl stop postfix
systemctl disable postfix

#7.关闭swap
echo "0" >  /proc/sys/vm/swappiness 
#8.开启转发
echo 1 > /proc/sys/net/ipv4/ip_forward
#9. 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
#10. ntp对时间
yum install ntpdate -y    
ntpdate -u cn.ntp.org.cn

echo "* * * * * ntpdate -u cn.ntp.org.cn"  >> /var/spool/cron/root 
#11.文件打开数
echo " *                soft    nofile       864000 " >>  /etc/security/limits.conf 
echo " *                hard    nofile       864000 " >>  /etc/security/limits.conf 

#12.安装docker-ce
yum remove docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-engine
				  
# Install using the repository
yum install -y yum-utils \
  device-mapper-persistent-data \
  lvm2

#快的镜像源
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

#这个比较慢  
yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo				  
yum install docker-ce docker-ce-cli containerd.io -y
systemctl start docker
systemctl enable docker


#13. 配置加速器
cat <<EOF > /etc/docker/daemon.json 
{
"registry-mirrors": [
     "https://1nj0zren.mirror.aliyuncs.com",
     "https://docker.mirrors.ustc.edu.cn",
     "http://f1361db2.m.daocloud.io",
     "https://registry.docker-cn.com"
    ]
}
EOF


#14. 配置日志切割
cat <<EOF >   /etc/logrotate.d/docker-logs 
/var/lib/docker/containers/*/*.log {
 rotate 7
 daily
 compress
 size=1M
 missingok
 delaycompress
 copytruncate
}
EOF

#15.安装 nfs 依赖
yum install nfs-utils rpcbind vim  -y  
systemctl start rpcbind
systemctl start nfs

node节点环境配置

#修改解析文件
# cat /etc/resolv.conf
search k8s.host.com
nameserver 10.65.10.1

#装包,kube-proxy 依赖包
yum -y install conntrack    

#修改主机名
# hostname
lgy-k8s-node0012.k8s.host.com

#操作dns主机记录
lgy-k8s-node0012.k8s IN A 10.65.0.12

#dns reload生效
/opt/named/sbin/rndc reload

#创建相关数据目录
mkdir -p /opt/kubernetes/server/bin/
mkdir -p /etc/kubernetes/pki
mkdir -p /etc/kubernetes/kubeconfig/
mkdir -p /export/kubernetes/logs/
mkdir -p /export/kubernetes/kubelet
mkdir -p /etc/flannel/pki/

master节点生成bootstrap

#在master节点 10.65.0.21 给Node节点创建bootstrap 文件
# cd  /opt/kubernetes/server/bin
cat > environment.sh <<EOF
#!/usr/bin/bash
KUBE_APISERVER="https://10.65.2.10:6443"
BOOTSTRAP_TOKEN="head -c 16 /dev/urandom | od -An -t x | tr -d ' '"
NODE_NAMES=(lgy-k8s-node0012 )
CLUSTER_DNS_SVC_IP="10.254.0.2"
CLUSTER_DNS_DOMAIN="cluster.local"
EOF

#source environment.sh
#node_name=${NODE_NAMES}
# 创建 token
export BOOTSTRAP_TOKEN=$(kubeadm token create \
--description kubelet-bootstrap-token \
--groups system:bootstrappers:${node_name} \
--kubeconfig ~/.kube/config)

#设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

# 设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap-${node_name}.kubeconfig

#分发bootstrap token 文件
scp kubelet-bootstrap-lgy-k8s-node0012.kubeconfig  10.65.0.12:/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig

#将kubelet、kube-proxy等文件拷贝至node节点
scp /opt/kubernetes/server/bin/kubelet  /opt/kubernetes/server/bin/kube-proxy root@10.65.0.12:/opt/kubernetes/server/bin/
scp /etc/kubernetes/pki/ca.pem root@10.65.0.12:/etc/kubernetes/pki/
scp  /usr/local/bin/flanneld root@10.65.0.12:/usr/local/bin/
scp /opt/kubernetes/server/bin/kube-proxy root@10.65.0.12:/opt/kubernetes/server/bin/
scp /etc/flannel/pki/flanneld.pem   10.65.0.12:/etc/flannel/pki/
scp /etc/flannel/pki/flanneld-key.pem  10.65.0.12:/etc/flannel/pki/
scp  /usr/local/bin/mk-docker-opts.sh   10.65.0.12:/usr/local/bin/
scp /etc/kubernetes/kubeconfig/kube-proxy.kubeconfig  10.65.0.12:/etc/kubernetes/kubeconfig/

node节点操作(lgy-k8s-node0012 )

#增加启动文件
# cat /etc/systemd/system/kubelet.service
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/opt/kubernetes/server/bin/kubelet \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBELET_API_SERVER \
            $KUBELET_ADDRESS \
            $KUBELET_PORT \
            $KUBELET_HOSTNAME \
            $KUBELET_POD_INFRA_CONTAINER \
            $KUBELET_ARGS
Restart=on-failure
RestartSec=5
StartLimitInterval=0

[Install]
WantedBy=multi-user.target


# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=false --log-dir=/export/kubernetes/logs/"
KUBE_LOG_LEVEL="--v=2"
KUBE_ALLOW_PRIV="--allow-privileged=true"

# cat /etc/kubernetes/kubelet
KUBELET_HOSTNAME="--hostname-override=lgy-k8s-node0012"  #修改为node节点本机主机名
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=harbor.k8s.moviebook.cn/library/pause:latest" #本地要配置镜像仓库，确保拉取镜像正常
KUBELET_ARGS="--cgroup-driver=systemd \
   --config=/etc/kubernetes/kubelet-config.yaml \
   --bootstrap-kubeconfig=/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig \
   --kubeconfig=/etc/kubernetes/kubeconfig/kubelet.kubeconfig \
   --cert-dir=/etc/kubernetes/pki \
   --root-dir=/export/kubernetes/kubelet \
   --image-pull-progress-deadline=15m"


# cat /etc/kubernetes/kubelet-config.yaml
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: "10.65.0.12"  #node节点ip地址
staticPodPath: ""
syncFrequency: 1m
fileCheckFrequency: 20s
httpCheckFrequency: 20s
staticPodURL: ""
port: 10250
readOnlyPort: 10255
rotateCertificates: true
serverTLSBootstrap: true
authentication:
  anonymous:
    enabled: false
  webhook:
    enabled: true
  x509:
    clientCAFile: "/etc/kubernetes/pki/ca.pem"
authorization:
  mode: Webhook
registryPullQPS: 0
registryBurst: 20
eventRecordQPS: 0
eventBurst: 20
enableDebuggingHandlers: true
enableContentionProfiling: true
healthzPort: 10248
healthzBindAddress: "10.65.0.12"
clusterDomain: "cluster.local"
clusterDNS:
  - "10.254.0.2"
nodeStatusUpdateFrequency: 10s
nodeStatusReportFrequency: 1m
imageMinimumGCAge: 2m
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
volumeStatsAggPeriod: 1m
kubeletCgroups: ""
systemCgroups: ""
cgroupRoot: ""
cgroupsPerQOS: true
cgroupDriver: cgroupfs
runtimeRequestTimeout: 10m
hairpinMode: promiscuous-bridge
maxPods: 50
podCIDR: "172.0.0.0/16"
podPidsLimit: -1
resolvConf: /etc/resolv.conf
maxOpenFiles: 1000000
kubeAPIQPS: 1000
kubeAPIBurst: 2000
serializeImagePulls: false
evictionHard:
  memory.available: "100Mi"
  nodefs.available: "10%"
  nodefs.inodesFree: "5%"
  imagefs.available: "15%"
evictionSoft: {}
enableControllerAttachDetach: true
failSwapOn: true
containerLogMaxSize: 20Mi
containerLogMaxFiles: 10
systemReserved: {}
kubeReserved: {}
systemReservedCgroup: ""
kubeReservedCgroup: ""
enforceNodeAllocatable: ["pods"]


#/etc/kubernetes/kubeconfig/kubelet-bootstrap.kubeconfig 在前面已生成
#/etc/kubernetes/kubeconfig/kubelet.kubeconfig 文件会重启kubelet自动生成
#systemctl start kubelet
#systemctl enable kubelet

#kube-proxy 安装
# cat /etc/systemd/system/kube-proxy.service 
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/opt/kubernetes/server/bin/kube-proxy \
            $KUBE_LOGTOSTDERR \
            $KUBE_LOG_LEVEL \
            $KUBE_PROXY_ARGS
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]

# cat /etc/kubernetes/config
KUBE_LOGTOSTDERR="--logtostderr=false --log-dir=/export/kubernetes/logs/"
KUBE_LOG_LEVEL="--v=2"
KUBE_ALLOW_PRIV="--allow-privileged=true"

# cat /etc/kubernetes/proxy
KUBE_PROXY_ARGS="--config=/etc/kubernetes/kube-proxy-config.yaml"

# cat /etc/kubernetes/kube-proxy-config.yaml
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clientConnection:
  burst: 200
  kubeconfig: "/etc/kubernetes/kubeconfig/kube-proxy.kubeconfig"
  qps: 100
bindAddress: 10.65.0.12
healthzBindAddress: 10.65.0.12:10256
metricsBindAddress: 10.65.0.12:10249
enableProfiling: true
clusterCIDR: 172.0.0.0/16
hostnameOverride: lgy-k8s-node0036
mode: "ipvs"
portRange: ""
iptables:
  masqueradeAll: false
ipvs:
  scheduler: nq
  excludeCIDRs: []



# systemctl start kube-proxy
#systemctl enable kube-proxy


#flannel 安装
# cat /etc/systemd/system/flannel.service 
[Unit]
Description=Flanneld overlay address etcd agent
After=network.target
After=network-online.target
Wants=network-online.target
After=etcd.service
Before=docker.service

[Service]
Type=notify
ExecStart=/usr/local/bin/flanneld \
  -etcd-cafile=/etc/kubernetes/pki/ca.pem \
  -etcd-certfile=/etc/flannel/pki/flanneld.pem \
  -etcd-keyfile=/etc/flannel/pki/flanneld-key.pem \
  -etcd-endpoints=https://10.65.10.1:2379,https://10.65.10.2:2379,https://10.65.10.4:2379 \
  -etcd-prefix=/etc/flannel/ \
  -iface=bond0 \
  -v=2
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/docker
Restart=on-failure
RestartSec=5
StartLimitInterval=0

[Install]
WantedBy=multi-user.target
RequiredBy=docker.service

#启动flannel
systemctl start flannel
systemctl enable flannel

#查看kubelet 日志
#journalctl -f -u kubelet 查看日志发现有 lgy-k8s-node0012 not found 信息输出，登录master节点
#master节点执行
kubectl get csr | awk '/Pending/ {print $1}' | xargs kubectl certificate approve

#查看node节点
kubectl get node

• 修改 docker 配置文件 (如果docker ip 地址和flannel 分配的不一致，需要修改docker 配置文件),重启docker 生效,还需要删除主机所有docker 容器，停止kubelet进行操作

#增加配置文件 EnvironmentFile=-/run/flannel/docker

#cat /usr/lib/systemd/system/docker.service

[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock
ExecReload=/bin/kill -s HUP $MAINPID
Environment="PATH=/opt/k8s/bin:/bin:/sbin:/usr/bin:/usr/sbin"
EnvironmentFile=-/run/flannel/docker
ExecStart=/usr/bin/dockerd $DOCKER_NETWORK_OPTIONS
TimeoutSec=0
RestartSec=2
Restart=always