AWS-自建集群K8S-Node节点初始化

节点初始化#

系统配置#

sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0


ln -snf /usr/share/zoneinfo/Asia/Riyadh /etc/localtime

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF


yum clean all
yum makecache

yum install -y kernel-devel make cmake gcc gcc-c++ openssl openssl-devel patch zlib zlib-devel pcre pcre-devel bash-completion net-tools bind-utils lrzsz wget lsof zip unzip sysstat iftop jq psmisc wget telnet vim git sudo tree curl htop ca-certificates net-snmp-utils nfs-utils tcpdump wget

cat >> /etc/profile << EOF
######################################
HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S  `whoami` "
EOF
source /etc/profile

yum install chrony -y
systemctl restart chronyd
systemctl enable chronyd



cat >> /etc/bashrc << EOF
alias grep='grep --color=auto'
alias egrep='egrep --color=auto'
EOF


cat >> /etc/security/limits.conf <<EOF
 *       soft    nofile  1024000
 *       hard    nofile  1024000
 *       soft    nproc   655350
 *       hard    nproc   655350
 *       soft    core    unlimited
 *       hard    core    unlimited
 * -   memlock unlimited
EOF

内核参数#

modprobe br_netfilter
chmod +x /etc/rc.d/rc.local
echo "modprobe br_netfilter" >> /etc/rc.local


cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
overlay
br_netfilter
EOF

modprobe overlay
modprobe br_netfilter
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack

# cat /etc/sysctl.conf
vm.swappiness = 0
vm.max_map_count=262144
##
net.ipv6.conf.lo.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv6.conf.all.disable_ipv6 = 1
##
net.ipv4.tcp_wmem=4096 12582912 16777216
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_slow_start_after_idle=0
net.ipv4.tcp_rmem=4096 12582912 16777216
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_max_tw_buckets = 5000
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_orphans = 16384
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.neigh.default.gc_stale_time = 120
net.ipv4.ip_local_reserved_ports = 30000-32767
net.ipv4.ip_local_port_range = 1024 65530
net.ipv4.ip_forward=1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.default.rp_filter = 0
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.all.arp_announce = 2
net.core.wmem_max=16777216
net.core.somaxconn = 32768
net.core.rmem_max=16777216
net.core.netdev_max_backlog=16384
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
##
kernel.sysrq = 1
kernel.softlockup_panic=1
kernel.softlockup_all_cpu_backtrace=1
kernel.pid_max = 327680
##
fs.inotify.max_user_watches=524288
fs.inotify.max_user_instances=8192
fs.inotify.max_queued_events=16384
fs.file-max=2097152

Docker#

yum install -y docker

• 通用参考配置（AWS 系统不适用）
• https://docs.docker.com/engine/install/centos/

cat /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "graph": "/data/docker",
  "insecure-registries": ["reg.jevic.cn"],
  "bridge": "none",
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "10m",
    "max-file": "10"
  },
  "bip": "169.254.123.1/24",
  "oom-score-adjust": -1000,
  "live-restore": true,
  "max-concurrent-downloads": 10,
  "default-ulimits": {
    "memlock": {
      "Hard": -1,
      "Name": "memlock",
      "Soft": -1
    }
  }
}

• AWS 节点配置

# aws 自身对docker 已经做了调整和优化
# cat /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "graph": "/data/docker",
  "insecure-registries": ["reg.jevic.cn"],
  "log-opts": {
    "max-size": "10m",
    "max-file": "10"
  },
  "bip": "169.254.123.1/24",
  "oom-score-adjust": -1000,
  "max-concurrent-downloads": 10
}

kubernetes#

• https://v1-23.docs.kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/

cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-\$basearch
enabled=1
gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
exclude=kubelet kubeadm kubectl
EOF

yum install -y kubelet-1.23.10  kubeadm-1.23.10  kubectl-1.23.10 --disableexcludes=kubernetes

• 替换kubeadm 为编译后的

# wget https://overseas-ftp.jevic.cn/aws_build_k8s/kubeadm_1.23.10.tar.gz
# tar zxf kubeadm_1.23.10.tar.gz && cd kubeadm_1.23.10
# mv /bin/kubeadm /bin/kubeadm.backup
# mv kubeadm /bin/
# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"21+", GitVersion:"v1.21.11-dirty", GitCommit:"38d3c1f3d5306401bcf39a71bad3b5a5106033d7", GitTreeState:"dirty", BuildDate:"2022-04-15T03:40:07Z", GoVersion:"go1.16.15", Compiler:"gc", Platform:"linux/amd64"}

• nginx-proxy
  • master 节点IP地址固定

# cat /etc/nginx/nginx.conf
error_log stderr notice;
worker_processes 4;
events {
  multi_accept on;
  use epoll;
  worker_connections 1024;
}

stream {
    upstream kube_apiserver {
        least_conn;
        server IP:6443;
        server IP:6443;
        server IP:6443;
    }

    server {
        listen        0.0.0.0:8443;
        proxy_pass    kube_apiserver;
        proxy_timeout 10m;
        proxy_connect_timeout 1s;
    }
}

# cat /etc/systemd/system/nginx-proxy.service
[Unit]
Description=kubernetes apiserver docker wrapper
Wants=docker.socket
After=docker.service

[Service]
User=root
PermissionsStartOnly=true
ExecStart=/usr/bin/docker run -p 127.0.0.1:8443:6443 \
                              -v /etc/nginx:/etc/nginx \
                              --name nginx-proxy \
                              --net=host \
                              --restart=on-failure:5 \
                              --memory=512M \
                              reg.jevic.cn/k8s/nginx:alpine
ExecStartPre=-/usr/bin/docker rm -f nginx-proxy
ExecStop=/usr/bin/docker stop nginx-proxy
Restart=always
RestartSec=15s
TimeoutStartSec=30s

[Install]
WantedBy=multi-user.target

---

以上是 k8s 节点的基础初始化部分内容