Centos7二进制部署k8s-v1.20.2 ipvs版本(kube-apiserver、kubelet)

一、kube-apiserver组件部署

获取最新更新以及文章用到的软件包，请移步点击：查看更新

1、下载安装包

#下载安装包
mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs} 
wget https://dl.k8s.io/v1.20.1/kubernetes-server-linux-amd64.tar.gz

#解压
tar -xf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/

#复制到安装目录
cp kube-apiserver kube-controller-manager kube-scheduler  /opt/kubernetes/bin
cp kubectl /usr/local/bin

#同步到其他节点
scp kube-apiserver kube-controller-manager kube-scheduler kubectl root@192.168.112.132:/opt/kubernetes/bin
scp kubectl root@192.168.112.132:/usr/local/bin

2、创建ca请求文件

cd /root/TLS/k8s
cat > ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
      "algo": "rsa",
      "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Sichuan",
      "L": "Chengdu",
      "O": "k8s",
      "OU": "system"
    }
  ],
  "ca": {
          "expiry": "175200h"
  }
}
EOF

#创建ca证书
cfssl gencert -initca ca-csr.json  | cfssljson -bare ca

注：
CN：Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名 (User Name)；浏览器使用该字段验证网站是否合法；
O：Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)

3、配置ca证书策略

cat > ca-config.json << EOF
{
  "signing": {
      "default": {
          "expiry": "175200h"
        },
      "profiles": {
          "kubernetes": {
              "usages": [
                  "signing",
                  "key encipherment",
                  "server auth",
                  "client auth"
              ],
              "expiry": "175200h"
          }
      }
  }
}
EOF

创建csr请求文件
cat > kube-apiserver-csr.json << EOF
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "192.168.112.131",
    "192.168.112.132",
    "192.168.112.133",
    "192.168.112.134",
    "192.168.112.135",
    "192.168.112.136",
    "192.168.112.137",
    "192.168.112.130",
    "10.255.0.1",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Sichuan",
      "L": "Chengdu",
      "O": "k8s",
      "OU": "system"
    }
  ]
}
EOF

生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver

注：
如果 hosts 字段不为空则需要指定授权使用该证书的 IP 或域名列表。
由于该证书后续被 kubernetes master 集群使用，需要将master节点的IP都填上，同时还需要填写 service 网络的首个IP。(一般是 kube-apiserver 指定的 service-cluster-ip-range 网段的第一个IP，如 10.254.0.1)

4、生成token文件

cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

5、创建配置文件

cat > /opt/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \\
  --anonymous-auth=false \\
  --bind-address=192.168.112.131 \\
  --secure-port=6443 \\
  --advertise-address=192.168.112.131 \\
  --authorization-mode=Node,RBAC \\
  --runtime-config=api/all=true \\
  --enable-bootstrap-token-auth \\
  --service-cluster-ip-range=10.255.0.0/16 \\
  --token-auth-file=/opt/kubernetes/cfg/token.csv \\
  --service-node-port-range=30000-50000 \\
  --tls-cert-file=/opt/kubernetes/ssl/kube-apiserver.pem  \\
  --tls-private-key-file=/opt/kubernetes/ssl/kube-apiserver-key.pem \\
  --client-ca-file=/opt/kubernetes/ssl/ca.pem \\
  --kubelet-client-certificate=/opt/kubernetes/ssl/kube-apiserver.pem \\
  --kubelet-client-key=/opt/kubernetes/ssl/kube-apiserver-key.pem \\
  --service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
  --service-account-signing-key-file=/opt/kubernetes/ssl/ca-key.pem  \\
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
  --etcd-cafile=/opt/etcd/ssl/ca.pem \\
  --etcd-certfile=/opt/etcd/ssl/etcd.pem \\
  --etcd-keyfile=/opt/etcd/ssl/etcd-key.pem \\
  --etcd-servers=https://192.168.112.131:2379,https://192.168.112.132:2379,https://192.168.112.133:2379 \\
  --enable-swagger-ui=true \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=30 \\
  --audit-log-maxbackup=3 \\
  --audit-log-maxsize=100 \\
  --audit-log-path=/opt/kubernetes/logs/kube-apiserver-audit.log \\
  --event-ttl=1h \\
  --alsologtostderr=true \\
  --logtostderr=false \\
  --log-dir=/opt/kubernetes/logs/kubernetes \\
  --v=4"
EOF

注：
–logtostderr：启用日志
–v：日志等级
–log-dir：日志目录
–etcd-servers：etcd集群地址
–bind-address：监听地址
–secure-port：https安全端口
–advertise-address：集群通告地址
–allow-privileged：启用授权
–service-cluster-ip-range：Service虚拟IP地址段
–enable-admission-plugins：准入控制模块
–authorization-mode：认证授权，启用RBAC授权和节点自管理
–enable-bootstrap-token-auth：启用TLS bootstrap机制
–token-auth-file：bootstrap token文件
–service-node-port-range：Service nodeport类型默认分配端口范围
–kubelet-client-xxx：apiserver访问kubelet客户端证书
–tls-xxx-file：apiserver https证书
–etcd-xxxfile：连接Etcd集群证书
–audit-log-xxx：审计日志

6、创建服务启动文件

cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service

[Service]
EnvironmentFile=-/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

直接复制文章中的，可能会丢失一个参数，记得手动添加$KUBE_APISERVER_OPTS，文章中其他涉及到像此处的，重复此操作

7、同步相关文件到各个节点

cp ca*.pem kube-apiserver*.pem /opt/kubernetes/ssl/
cp token.csv /opt/kubernetes/cfg/

scp /usr/lib/systemd/system/kube-apiserver.service root@192.168.112.132:/usr/lib/systemd/system/
scp ca*.pem kube-apiserver*.pem root@192.168.112.132:/opt/kubernetes/ssl/
scp token.csv root@192.168.112.132:/opt/kubernetes/cfg/

注：master2和master3配置文件的IP地址修改为实际的本机IP

启动服务
systemctl daemon-reload
systemctl enable kube-apiserver
systemctl restart kube-apiserver
systemctl status kube-apiserver
测试
curl --insecure https://192.168.112.131:6443/
有返回说明启动正常

二、部署kubectl组件

1、创建csr请求文件

cat > admin-csr.json << EOF
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Sichuan",
      "L": "Chengdu",
      "O": "system:masters",             
      "OU": "system"
    }
  ]
}
EOF

#生成证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

#同步证书
cp admin*.pem /opt/kubernetes/ssl/
scp admin*.pem root@192.168.112.132:/opt/kubernetes/ssl/

说明：
后续 kube-apiserver 使用 RBAC 对客户端(如 kubelet、kube-proxy、Pod)请求进行授权；
kube-apiserver 预定义了一些 RBAC 使用的 RoleBindings，如 cluster-admin 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予了调用kube-apiserver 的所有 API的权限；
O指定该证书的 Group 为 system:masters，kubelet 使用该证书访问 kube-apiserver 时 ，由于证书被 CA 签名，所以认证通过，同时由于证书用户组为经过预授权的 system:masters，所以被授予访问所有 API 的权限；
注：
这个admin 证书，是将来生成管理员用的kube config 配置文件用的，现在我们一般建议使用RBAC 来对kubernetes 进行角色权限控制， kubernetes 将证书中的CN 字段 作为User， O 字段作为 Group；
“O”: “system:masters”, 必须是system:masters，否则后面kubectl create clusterrolebinding报错。

2、创建kubeconfig配置文件

kubeconfig 为 kubectl 的配置文件，包含访问 apiserver 的所有信息，如 apiserver 地址、CA 证书和自身使用的证书

设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=ca.pem --embed-certs=true --server=https://192.168.112.130:7443 --kubeconfig=kube.config
设置客户端认证参数
kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config
设置上下文参数
kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config
设置默认上下文
kubectl config use-context kubernetes --kubeconfig=kube.config
mkdir ~/.kube
cp kube.config ~/.kube/config

同步kubectl配置文件到其他节点
scp -r /root/.kube root@192.168.112.132:/root/

四、安装nginx和keepalived，对apiserver做高可用负载

1、在两台master节点安装nginx

#编辑yum文件
vim /etc/yum.repos.d/nginx.repo
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

#安装最新版nginx
yum install nginx -y

#编辑nginx配置文件，nginx四层负载，必须与http同级
vi /etc/nginx/nginx.conf

stream {
    upstream kube-apiserver {
        server 192.168.112.131:6443     max_fails=3 fail_timeout=30s;
        server 192.168.112.132:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 7443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}

#启动nginx
nginx -t
systemctl start nginx
systemctl enable nginx

2、部署keepalived实现高可用

#安装keepalived 
yum install keepalived -y

#编写keepalived监控脚本
vi /etc/keepalived/check_port.sh
#!/bin/bash
#keepalived 监控端口脚本
#使用方法：
#在keepalived的配置文件中
#vrrp_script check_port {#创建一个vrrp_script脚本,检查配置
#    script "/etc/keepalived/check_port.sh 6379" #配置监听的端口
#    interval 2 #检查脚本的频率,单位（秒）
#}
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
        PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`
        if [ $PORT_PROCESS -eq 0 ];then
                echo "Port $CHK_PORT Is Not Used,End."
                exit 1
        fi
else
        echo "Check Port Cant Be Empty!"
fi

#对监控脚本授权
chmod +x /etc/keepalived/check_port.sh

#编辑keepalived配置文件，注意主从配置文件不一样
vi /etc/keepalived/keepalived.conf
主节点:
! Configuration File for keepalived
 
global_defs {
   router_id 192.168.112.131
 
}
 
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}
 
vrrp_instance VI_1 {
    state MASTER
    interface ens192                       修改网卡名字
    virtual_router_id 251
    priority 100
    advert_int 1
    mcast_src_ip 192.168.112.131
    nopreempt   #非抢占式 ，当主节点挂了以后，从节点vip飘到从上，主节点恢复以后，不主动飘回主，需要手动重启keepalived
 
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
         chk_nginx
    }
    virtual_ipaddress {
        192.168.112.130
    }
}

从节点：
! Configuration File for keepalived
global_defs {
    router_id  192.168.112.132
}
vrrp_script chk_nginx {
    script "/etc/keepalived/check_port.sh 7443"
    interval 2
    weight -20
}
vrrp_instance VI_1 {
    state BACKUP
    interface ens192                     修改网卡名字
    virtual_router_id 251
    mcast_src_ip  192.168.112.132
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 11111111
    }
    track_script {
        chk_nginx
    }
    virtual_ipaddress {
        192.168.112.130
    }
}

#启动keepalived并配置开机自启
systemctl start keepalived
systemctl enable keepalived

#检查ip地址
[root@clihouse01 ~]# ip -4 a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    inet 192.168.112.131/24 brd 192.168.112.255 scope global noprefixroute ens192
       valid_lft forever preferred_lft forever
    inet 192.168.112.130/32 scope global ens192                        会多一个VIP
       valid_lft forever preferred_lft forever

五、验证集群

#授权kubernetes证书访问kubelet api权限
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

#查看集群组件状态
kubectl cluster-info
kubectl get componentstatuses
kubectl get all --all-namespaces