k8s、jumpserver

一、安装jumpserver

官网：https://docs.jumpserver.org/zh/master/install/setup_by_fast/

安装docker

 #! /bin/bash
apt update
 
apt install \
    ca-certificates \
    curl \
    gnupg \
    lsb-release
 
mkdir -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
 
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
 
apt update
apt -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin
 
sed -i '/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/c GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 cgroup_enable=memory swapaccount=1"' /etc/default/grub
update-grub
reboot

查看docker

 [root@jumpserver ~]#docker --version
Docker version 20.10.17, build 100c701

下载镜像

 docker pull mysql:5.6
docker pull redis:6.0
docker pull jumpserver/jms_all:1.5.9

查看镜像

 [root@jumpserver ~]#docker images
REPOSITORY   TAG       IMAGE ID       CREATED        SIZE
redis                6.0       6487e6a1b120   2 weeks ago    112MB
mysql                5.6       dd3b2a5dcb48   8 months ago   303MB
jumpserver/jms_all   1.5.9     6573e4b624c9   2 years ago    1.48GB

部署mysql

准备配置文件

创建目录

 mkdir -p /data/mysql
mkdir -p /data/etc/mysql/conf.d
mkdir -p /data/etc/mysql/mysql.conf.d

mysql.cnf配置文件

 [root@jumpserver ~]#cat /data/etc/mysql/conf.d/mysql.cnf
[mysql]
default-character-set=utf8

mysqld.cnf配置文件

 [root@jumpserver ~]#cat /data/etc/mysql/mysql.conf.d/mysqld.cnf
[mysqld]
pid-file	= /var/run/mysqld/mysqld.pid
socket		= /var/run/mysqld/mysqld.sock
datadir		= /var/lib/mysql
log-error	= /var/log/mysql/error.log
symbolic-links=0
character-set-server=utf8

目录结构如下

 [root@jumpserver ~]#tree /data
/data
├── etc
│   └── mysql
│       ├── conf.d
│       │   └── mysql.cnf
│       └── mysql.conf.d
│           └── mysqld.cnf
└── mysql
 
5 directories, 2 files

创建mysql容器

 docker run -it -d -p 3306:3306 -v /data/etc/mysql/conf.d/mysql.cnf:/etc/mysql/conf.d/mysql.cnf \
-v /data/etc/mysql/mysql.conf.d/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf \
-v /data/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD='123456' mysql:5.6

验证数据库连接

 # 宿主机连接，先安装apt install mysql-client
[root@jumpserver mysql]#mysql -uroot -p123456 -h10.0.0.12
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.6.51 MySQL Community Server (GPL)
 
Copyright (c) 2000, 2022, Oracle and/or its affiliates.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
+--------------------+
3 rows in set (0.01 sec)

验证数据库编码（utf8）

 mysql> show variables like "%character%";
+--------------------------+----------------------------+
| Variable_name            | Value                      |
+--------------------------+----------------------------+
| character_set_client     | utf8                       |
| character_set_connection | utf8                       |
| character_set_database   | utf8                       |
| character_set_filesystem | binary                     |
| character_set_results    | utf8                       |
| character_set_server     | utf8                       |
| character_set_system     | utf8                       |
| character_sets_dir       | /usr/share/mysql/charsets/ |
+--------------------------+----------------------------+
8 rows in set (0.01 sec)
 
mysql> show variables like "%collation%";
+----------------------+-----------------+
| Variable_name        | Value           |
+----------------------+-----------------+
| collation_connection | utf8_general_ci |
| collation_database   | utf8_general_ci |
| collation_server     | utf8_general_ci |
+----------------------+-----------------+
3 rows in set (0.00 sec)

创建jumpserver数据库，并授权jumpserver用户

注意：授权密码不能为纯数字

 [root@jumpserver data]#mysql -uroot -p123456 -h10.0.0.12
 
mysql> create database jumpserver default charset 'utf8';
Query OK, 1 row affected (0.00 sec)
 
mysql> grant all on jumpserver.* to 'jumpserver'@'%' identified by 'jp123456';
Query OK, 0 rows affected (0.00 sec)

验证jumpserver用户权限

 [root@jumpserver mysql.conf.d]#mysql -ujumpserver -pjp123456 -h10.0.0.12
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 2
Server version: 5.6.51 MySQL Community Server (GPL)
 
Copyright (c) 2000, 2022, Oracle and/or its affiliates.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| jumpserver         |
+--------------------+
2 rows in set (0.00 sec)

部署redis

创建redis容器

 [root@jumpserver mysql]#docker run -d -it -p 6379:6379 redis:6.0
331baf1779e94bc110f4d15fad2e0bce743ff17c051862216bc1d629ba47177c

验证redis访问

 [root@jumpserver mysql]#redis-cli -h 10.0.0.12
10.0.0.12:6379> info
# Server
redis_version:6.0.16
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:b9d515b0d5e963de
redis_mode:standalone
os:Linux 5.4.0-122-generic x86_64
arch_bits:64

部署jumpserver

生成随机加密密钥和初始化token

 # 随机密钥
cat /dev/urandom |tr -dc A-Za-z0-9|head -c 50 >> /data/secret_key
j3GFsi5Qwcn5Jj4rugfwvdBc56B6QMkyBuncwRP72XviP7mxTU
 
# token
cat /dev/urandom |tr -dc A-Za-z0-9|head -c 16 >> bootstrap_token
kiBEmIVpWrFrlY3q

创建jumpserver容器

 docker run -d --name jms_all \
-v /data/jumpserver:/opt/jumpserver/data/media \
-p 80:80 \
-p 2222:2222 \
-e SECRET_KEY=j3GFsi5Qwcn5Jj4rugfwvdBc56B6QMkyBuncwRP72XviP7mxTU \
-e BOOTSTRAP_TOKEN=kiBEmIVpWrFrlY3q \
-e DB_HOST=10.0.0.12 \
-e DB_PORT=3306 \
-e DB_USER='jumpserver' \
-e DB_PASSWORD='jp123456' \
-e DB_NAME=jumpserver \
-e REDIS_HOST=10.0.0.12 \
-e REDIS_PORT=6379 \
-e REDIS_PASSWORD= \
jumpserver/jms_all:1.5.9

启动后需等待一段时间

 Starting guacd: guacd[91]: INFO:	Guacamole proxy daemon (guacd) version 1.0.0 started
SUCCESS
Tomcat started.
Jumpserver ALL 1.5.9
官网 http://www.jumpserver.org
文档 http://docs.jumpserver.org
有问题请参考 http://docs.jumpserver.org/zh/docs/faq.html
 
进入容器命令 docker exec -it jms_all /bin/bash

查看数据库内容

 [root@jumpserver ~]#mysql -ujumpserver -pjp123456 -h10.0.0.12
mysql: [Warning] Using a password on the command line interface can be insecure.
Welcome to the MySQL monitor.  Commands end with ; or \g.
Your MySQL connection id is 24
Server version: 5.6.51 MySQL Community Server (GPL)
 
Copyright (c) 2000, 2022, Oracle and/or its affiliates.
 
Oracle is a registered trademark of Oracle Corporation and/or its
affiliates. Other names may be trademarks of their respective
owners.
 
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
 
mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| jumpserver         |
+--------------------+
2 rows in set (0.06 sec)
 
mysql> use jumpserver;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
 
Database changed
mysql> show tables;
+----------------------------------------------+
| Tables_in_jumpserver                         |
+----------------------------------------------+
| applications_databaseapp                     |
| applications_remoteapp                       |
| assets_adminuser                             |
| assets_asset                                 |
| assets_asset_labels                          |
| assets_asset_nodes                           |
| assets_assetgroup                            |
| assets_authbook                              |
| assets_cluster                               |

登录web界面

默认用户名：admin

密码：admin

进入首页

二、配置用户tom，并添加一个相应的服务器资源

创建规则

创建用户

用户管理--用户列表

添加登录jumpserver用户信息
创建用户组

用户管理--用户组

添加组信息，将tom加入该组中
创建管理用户

登录服务器的管理员账户

添加管理员账户密码
创建系统用户

该系统用户是用于登录服务器的普通账户

资产管理--系统用户

添加系统用户信息

添加系统用户

资产管理--系统用户，选择www用户--资料列表，点击推送

执行推送情况
创建资产

资产包括服务器、虚拟机等

资产管理--资产列表，可新建web节点

添加服务器信息，可加入web节点

选中相应主机名，查看资产信息

点击测试

查看测试结果
创建授权规则

权限管理--资产授权

添加授权信息

将用户组、资产节点、系统用户进行关联

登录测试

jumpserver普通用户登录

输入tom用户名、密码
查看资产

我的资产--web节点--服务器

从动作进入服务器终端

或者点击Web终端
远程登录服务器

执行命令正常

三、使用命令过滤器禁止tom用户在服务器上使用 reboot、init、rm -rf、poweroff 这几个命令

创建过滤器

资产管理--命令过滤

 新建过滤器名称
 
<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210807192-729271614.png" width="50%" height="50%" />
 
 
点击过滤器名称
 
<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210814463-1057751346.png" width="50%" height="50%" />
 
 
创建规则
 
<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210843768-62335829.png" width="50%" height="50%" />
 
配置过滤器具体规则
 
<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210835210-274524939.png" width="50%" height="50%" />

使用过滤器

资产管理--系统用户

选择系统用户，点击更新

新添加创建好的命令过滤器
验证命令过滤器生效

无法执行reboot、poweroff、rm -rf等命令，其余命令执行正常

四、部署k8s指定版本1.20高可用集群

架构

安装高可用反向代理

基于haproxy+keepalived实现

安装keepalived

节点1（10.0.0.51）

安装keepalived

 apt install keepalived -y

keepalived配置文件

 root@haproxy1 keepalived]#vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id 10.0.0.51
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
   vrrp_mcast_group4 224.0.0.18
}
 
# 脚本需放在调用之前，先定义好
vrrp_script chk_haproxy {
    script "/etc/keepalived/chk_haproxy.sh"
    interval 1
    timeout 2
    weight -30
    fall 3
    rise 5
}
 
vrrp_instance haproxy {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {                                                                                                                                                                           
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.53/24 dev eth0 label eth0:1
    }
  
    track_script {
      chk_haproxy
    }
 
}

chk_haproxy脚本

 [root@haproxy1 keepalived#vim /etc/keepalived/chk_haproxy.sh
#!/bin/bash
/usr/bin/killall -0 haproxy
 
#添加执行权限
[root@haproxy1 keepalived# chmod a+x /etc/keepalived/chk_haproxy.sh

重启服务

 systemctl restart keepalived

查看VIP

  
[root@haproxy1 keepalived]#ifconfig  -a
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.51  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::20c:29ff:fef9:b4ed  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f9:b4:ed  txqueuelen 1000  (Ethernet)
        RX packets 20377  bytes 23085737 (23.0 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 11683  bytes 1239148 (1.2 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
 
eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.53  netmask 255.255.255.0  broadcast 0.0.0.0
        ether 00:0c:29:f9:b4:ed  txqueuelen 1000  (Ethernet)
 
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

节点2（10.0.0.52）

安装keepalived

 [root@haproxy2 ~]#apt install keepalived -y

keepalived配置文件

 [root@haproxy2 ~]#vim /etc/keepalived/keepalived.conf
 
! Configuration File for keepalived
 
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id 10.0.0.52			#与节点1不同，为本机IP
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
   vrrp_mcast_group4 224.0.0.18
}
 
vrrp_instance haproxy {
    state BACKUP			#BACKUP
    interface eth0
    virtual_router_id 51
    priority 80                         #优先级80                                                                                                                                                        
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.0.0.10/24 dev eth0 label eth0:1
    }
}

chk_haproxy脚本

 [root@haproxy2 ~]#vim /etc/keepalived/chk_haproxy.sh
#!/bin/bash
/usr/bin/killall -0 haproxy
 
#添加执行权限
[root@haproxy2 ~]#chmod a+x /etc/keepalived/chk_haproxy.sh

重启服务

 [root@haproxy2 ~]#systemctl restart keepalived

查看VIP

注意：需先停止节点1的keepalived服务

 [root@haproxy2 ~]#ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.52  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::20c:29ff:fe3d:976d  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:3d:97:6d  txqueuelen 1000  (Ethernet)
        RX packets 3371  bytes 3424676 (3.4 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 2136  bytes 253767 (253.7 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
 
eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.53  netmask 255.255.255.0  broadcast 0.0.0.0
        ether 00:0c:29:3d:97:6d  txqueuelen 1000  (Ethernet)
 
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

安装haproxy

节点1（10.0.0.51）

安装

 [root@haproxy1 ~]#apt install haproxy -y

修改配置文件

 [root@haproxy1 ~]#vim /etc/haproxy/haproxy.cfg
....
#末尾添加如下配置
listen stats
    mode http
    bind 0.0.0.0:9999
    stats enable
    log global
    stats uri   /haproxy-status
    stats auth  haadmin:123456
 
listen k8s-6443
    bind 10.0.0.53:6443
    mode tcp
    balance roundrobin
    server k8s-master1 10.0.0.54:6443 check inter 3s fall 3 rise 5
    server k8s-master2 10.0.0.55:6443 check inter 3s fall 3 rise 5
    server k8s-master3 10.0.0.56:6443 check inter 3s fall 3 rise 5

启用内核参数

 #限制响应级别：arp_ignore
#0：默认值，表示可使用本地任意接口上配置的任意地址进行响应
#1：仅在请求的目标IP配置在本地主机的接收到请求报文的接口上时，才给予响应
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
 
#限制通告级别：arp_announce
#0：默认值，把本机所有接口的所有信息向每个接口的网络进行通告
#1：尽量避免将接口信息向非直接连接网络进行通告
#2：必须避免将接口信息向非本网络进行通告
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
 
echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf	#开启后VIP不在本地，haproxy也可绑定该地址
echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf		#开启ipv4路由转发功能

执行sysctl -p命令，修改内核生效

重启服务

 root@haproxy1 ~]#systemctl restart haproxy

查看IP

 [root@haproxy1 ~]#ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.51  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::20c:29ff:fef9:b4ed  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:f9:b4:ed  txqueuelen 1000  (Ethernet)
        RX packets 24898  bytes 25024044 (25.0 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 15962  bytes 1611934 (1.6 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
 
eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.53  netmask 255.255.255.0  broadcast 0.0.0.0
        ether 00:0c:29:f9:b4:ed  txqueuelen 1000  (Ethernet)
 
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

查看端口，haproxy绑定VIP 6443端口正常

 [root@haproxy1 ~]#netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name  
tcp        0      0 10.0.0.53:6443          0.0.0.0:*               LISTEN      7783/haproxy  
tcp        0      0 0.0.0.0:9999            0.0.0.0:*               LISTEN      7783/haproxy  
tcp        0      0 0.0.0.0:111             0.0.0.0:*               LISTEN      1/init    
tcp        0      0 0.0.0.0:46961           0.0.0.0:*               LISTEN      824/rpc.mountd  
tcp        0      0 127.0.0.53:53           0.0.0.0:*               LISTEN      822/systemd-resolve 
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      915/sshd: /usr/sbin 
tcp        0      0 127.0.0.1:6010          0.0.0.0:*               LISTEN      1493/sshd: root@pts 
tcp        0      0 0.0.0.0:2049            0.0.0.0:*               LISTEN      -         
tcp        0      0 0.0.0.0:35845           0.0.0.0:*               LISTEN      824/rpc.mountd  
tcp        0      0 0.0.0.0:37381           0.0.0.0:*               LISTEN      824/rpc.mountd  
tcp        0      0 0.0.0.0:44905           0.0.0.0:*               LISTEN      -         
tcp6       0      0 :::37675                :::*                    LISTEN      -         
tcp6       0      0 :::111                  :::*                    LISTEN      1/init    
tcp6       0      0 :::22                   :::*                    LISTEN      915/sshd: /usr/sbin 
tcp6       0      0 ::1:6010                :::*                    LISTEN      1493/sshd: root@pts 
tcp6       0      0 :::47389                :::*                    LISTEN      824/rpc.mountd  
tcp6       0      0 :::42335                :::*                    LISTEN      824/rpc.mountd  
tcp6       0      0 :::54751                :::*                    LISTEN      824/rpc.mountd  
tcp6       0      0 :::2049                 :::*                    LISTEN      -

节点2（10.0.0.52）

安装

 [root@haproxy1 ~]#apt install haproxy -y

修改配置文件

 [root@haproxy1 ~]#vim /etc/haproxy/haproxy.cfg
....
#末尾添加如下配置
listen stats
    mode http
    bind 0.0.0.0:9999
    stats enable
    log global
    stats uri   /haproxy-status
    stats auth  haadmin:123456
 
listen k8s-6443			#新增以下配置
    bind 10.0.0.53:6443
    mode tcp
    balance roundrobin
    server k8s-master1 10.0.0.54:6443 check inter 3s fall 3 rise 5
    server k8s-master2 10.0.0.55:6443 check inter 3s fall 3 rise 5
    server k8s-master3 10.0.0.56:6443 check inter 3s fall 3 rise 5

启用内核参数

 #限制响应级别：arp_ignore
#0：默认值，表示可使用本地任意接口上配置的任意地址进行响应
#1：仅在请求的目标IP配置在本地主机的接收到请求报文的接口上时，才给予响应
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
 
#限制通告级别：arp_announce
#0：默认值，把本机所有接口的所有信息向每个接口的网络进行通告
#1：尽量避免将接口信息向非直接连接网络进行通告
#2：必须避免将接口信息向非本网络进行通告
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
 
echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf	#开启后VIP不在本地，haproxy也可绑定该地址
echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf		#开启ipv4路由转发功能

重启服务

 root@haproxy1 ~]#systemctl restart haproxy

查看IP

 [root@haproxy2 ~]#ifconfig 
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.52  netmask 255.255.255.0  broadcast 10.0.0.255
        inet6 fe80::20c:29ff:fe3d:976d  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:3d:97:6d  txqueuelen 1000  (Ethernet)
        RX packets 7089  bytes 5315138 (5.3 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 6911  bytes 687192 (687.1 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
 
lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

查看端口

 [root@haproxy2 ~]#netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name  
tcp        0      0 10.0.0.53:6443          0.0.0.0:*               LISTEN      7999/haproxy  
tcp        0      0 0.0.0.0:9999            0.0.0.0:*               LISTEN      7999/haproxy  
tcp        0      0 0.0.0.0:111             0.0.0.0:*               LISTEN      1/init    
tcp        0      0 0.0.0.0:46547           0.0.0.0:*               LISTEN      -         
tcp        0      0 127.0.0.53:53           0.0.0.0:*               LISTEN      812/systemd-resolve 
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      906/sshd: /usr/sbin 
tcp        0      0 127.0.0.1:6010          0.0.0.0:*               LISTEN      1451/sshd: root@pts 
tcp        0      0 0.0.0.0:45693           0.0.0.0:*               LISTEN      814/rpc.mountd  
tcp        0      0 0.0.0.0:60991           0.0.0.0:*               LISTEN      814/rpc.mountd  
tcp        0      0 0.0.0.0:40063           0.0.0.0:*               LISTEN      814/rpc.mountd  
tcp        0      0 0.0.0.0:2049            0.0.0.0:*               LISTEN      -         
tcp6       0      0 :::111                  :::*                    LISTEN      1/init    
tcp6       0      0 :::44277                :::*                    LISTEN      814/rpc.mountd  
tcp6       0      0 :::22                   :::*                    LISTEN      906/sshd: /usr/sbin 
tcp6       0      0 ::1:6010                :::*                    LISTEN      1451/sshd: root@pts 
tcp6       0      0 :::35711                :::*                    LISTEN      -         
tcp6       0      0 :::2049                 :::*                    LISTEN      -         
tcp6       0      0 :::38787                :::*                    LISTEN      814/rpc.mountd  
tcp6       0      0 :::58725                :::*                    LISTEN      814/rpc.mountd

查看haproxy监控状态页

访问10.0.0.51/10.0.0.52和VIP10.0.0.53

安装harbor镜像仓库

安装方法见

第五节-安装docker镜像仓库harbor,并实现高可用

登录web页面

新建test项目

上传镜像

 [root@harbor harbor]#docker push 10.0.0.60/test/alpine:v1
The push refers to repository [10.0.0.60/test/alpine]
994393dc58e7: Pushed 
v1: digest: sha256:1304f174557314a7ed9eddb4eab12fed12cb0cd9809e4c28f29af86979a3c870 size: 528

所有节点安装kubeadm组件

包括master节点和node节点

版本选择：https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md#v1205

安装docker

 #! /bin/bash
apt update
 
# 安装依赖包
apt install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg \
    lsb-release \
    software-properties-common
 
# 安装GPG证书
curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
 
sudo add-apt-repository "deb [arch=$(dpkg --print-architecture)] http://mirrors.aliyun.com/docker-ce/linux/ubuntu \
  $(lsb_release -cs) stable"
 
apt update
# apt-cache madison docker-ce docker-ce-cli
apt -y install docker-ce=5:19.03.15~3-0~ubuntu-$(lsb_release -cs) \
docker-ce-cli=5:19.03.15~3-0~ubuntu-$(lsb_release -cs)

系统优化

master节点CPU数量至少2

 # 关闭防火墙
systemctl disable firewalld && systemctl stop firewalld
# 在/etc/hosts中添加IP、主机名
cat >> /etc/hosts <<EOF
`hostname -I|awk '{print $1}'` `hostname`
EOF
 
# 内核参数优化
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
 
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
 
sudo sysctl --system
 
# 设置docker的cgroup driver
# docker 默认的 cgroup driver 是 cgroupfs，可以通过 docker info 命令查看
# 如果用户没有在 KubeletConfiguration 下设置 cgroupDriver 字段，则 kubeadm 将默认为systemd，需要将docker cgroup driver更改为systemd
cat <<EOF >/etc/docker/daemon.json
{
    "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
 
# 关闭swap
# 在/etc/fstab注释swap那一行
sed -ri 's/(^[^#]*swap)/#\1/' /etc/fstab
echo 'swapoff -a' >> /etc/profile
swapoff -a
 
# 修改grub
sed -i '/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/c GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 cgroup_enable=memory swapaccount=1"' /etc/default/grub
update-grub
reboot

设置国内阿里云镜像源

 apt update && apt-get install -y apt-transport-https
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - 
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
deb https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial main
EOF

安装kubelet、kubeadm、kubectl

 apt update
# 查看版本
# apt-cache madison kubeadm
 
# 安装1.20.5版本kubeadm、kubelet、kubectl
apt install -y kubeadm=1.20.5-00 kubelet=1.20.5-00 kubectl=1.20.5-00
 
# 查看版本
[root@k8s-master1 ~]#kubeadm version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:08:27Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}

目前kubelet启动报错

 [root@k8s-master1 ~]#systemctl status kubelet.service 
● kubelet.service - kubelet: The Kubernetes Node Agent
     Loaded: loaded (/lib/systemd/system/kubelet.service; enabled; vendor preset: enabled)
    Drop-In: /etc/systemd/system/kubelet.service.d
             └─10-kubeadm.conf
     Active: activating (auto-restart) (Result: exit-code) since Wed 2022-08-24 23:27:21 CST; 456ms ago
       Docs: https://kubernetes.io/docs/home/
    Process: 3707 ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS (code=exited, status=255/EXCEPTION)
   Main PID: 3707 (code=exited, status=255/EXCEPTION)
 
Aug 24 23:27:21 k8s-master1 systemd[1]: kubelet.service: Main process exited, code=exited, status=255/EXCEPTION
Aug 24 23:27:21 k8s-master1 systemd[1]: kubelet.service: Failed with result 'exit-code'.

kubeadm init集群初始化

官网：https://kubernetes.io/zh-cn/docs/reference/setup-tools/kubeadm/kubeadm-init/

准备镜像

 # 查看版本
[root@k8s-master1 ~]#kubeadm version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:08:27Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}
 
# 查看当前版本所需镜像
[root@k8s-master1 ~]#kubeadm config images list --kubernetes-version v1.20.5
k8s.gcr.io/kube-apiserver:v1.20.5
k8s.gcr.io/kube-controller-manager:v1.20.5
k8s.gcr.io/kube-scheduler:v1.20.5
k8s.gcr.io/kube-proxy:v1.20.5
k8s.gcr.io/pause:3.2
k8s.gcr.io/etcd:3.4.13-0
k8s.gcr.io/coredns:1.7.0

下载master节点镜像

 # 先提前下载特定的 Kubernetes 版本镜像，减少安装等待时间，镜像默认使用Google镜像仓库，国内无法直接下载
# 可使用阿里云镜像仓库把镜像提前下载下来，可以避免后期因镜像下载异常而导致k8s部署异常
[root@k8s-master1 ~]#kubeadm config images pull --kubernetes-version v1.20.5 \
--image-repository registry.cn-hangzhou.aliyuncs.com/google_containers

查看镜像

 [root@k8s-master1 ~]#docker images
REPOSITORY                                                                    TAG        IMAGE ID       CREATED         SIZE
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy                v1.20.5    5384b1650507   17 months ago   118MB
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler            v1.20.5    8d13f1db8bfb   17 months ago   47.3MB
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver            v1.20.5    d7e24aeb3b10   17 months ago   122MB
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager   v1.20.5    6f0c3da8c99e   17 months ago   116MB
registry.cn-hangzhou.aliyuncs.com/google_containers/etcd                      3.4.13-0   0369cf4303ff   24 months ago   253MB
registry.cn-hangzhou.aliyuncs.com/google_containers/coredns                   1.7.0      bfe3a36ebd25   2 years ago     45.2MB
registry.cn-hangzhou.aliyuncs.com/google_containers/pause                     3.2        80d28bedfe5d   2 years ago     683kB

高可用master初始化

基于keepalived实现高可用VIP，通过haproxy实现kube-apiserver的反向代理，将kube-apiserver的管理请求转发至多台k8s-master以实现管理端高可用。

注意：选择master1进行集群初始化，集群初始化只要初始化一次

kubeadm --init参数说明

--apiserver-advertise-address string	API 服务器所公布的其正在监听的 IP 地址。如果未设置，则使用默认网络接口。
--apiserver-bind-port int32 默认值：6443	API 服务器绑定的端口。
--apiserver-cert-extra-sans strings	用于 API Server 服务证书的可选附加主题备用名称（SAN）。可以是 IP 地址和 DNS 名称。
--cert-dir string 默认值："/etc/kubernetes/pki"	保存和存储证书的路径。
--certificate-key string	用于加密 kubeadm-certs Secret 中的控制平面证书的密钥。
--config string	kubeadm 配置文件的路径。
--control-plane-endpoint string	为控制平面指定一个稳定的 IP 地址或 DNS 名称。
--cri-socket string	要连接的 CRI 套接字的路径。如果为空，则 kubeadm 将尝试自动检测此值；仅当安装了多个 CRI 或具有非标准 CRI 插槽时，才使用此选项。
--dry-run	不要应用任何更改；只是输出将要执行的操作。
--feature-gates string	一组用来描述各种功能特性的键值（key=value）对。选项是：PublicKeysECDSA=true\|false (ALPHA - 默认值=false)RootlessControlPlane=true\|false (ALPHA - 默认值=false)UnversionedKubeletConfigMap=true\|false (BETA - 默认值=true)
-h, --help	init 操作的帮助命令
--ignore-preflight-errors strings	错误将显示为警告的检查列表；例如：'IsPrivilegedUser,Swap'。取值为 'all' 时将忽略检查中的所有错误。
--image-repository string 默认值："k8s.gcr.io"	选择用于拉取控制平面镜像的容器仓库
--kubernetes-version string 默认值："stable-1"	为控制平面选择一个特定的 Kubernetes 版本。
--node-name string	指定节点的名称。
--patches string	它包含名为 "target[suffix][+patchtype].extension" 的文件的目录的路径。例如，"kube-apiserver0+merge.yaml"或仅仅是 "etcd.json"。 "target" 可以是 "kube-apiserver"、"kube-controller-manager"、"kube-scheduler"、"etcd" 之一。 "patchtype" 可以是 "strategic"、"merge" 或者 "json" 之一，并且它们与 kubectl 支持的补丁格式相同。默认的 "patchtype" 是 "strategic"。 "extension" 必须是"json" 或"yaml"。 "suffix" 是一个可选字符串，可用于确定首先按字母顺序应用哪些补丁。
--pod-network-cidr string	指明 pod 网络可以使用的 IP 地址段。如果设置了这个参数，控制平面将会为每一个节点自动分配 CIDRs。
--service-cidr string 默认值："10.96.0.0/12"	为服务的虚拟 IP 地址另外指定 IP 地址段
--service-dns-domain string 默认值："cluster.local"	为服务另外指定域名，例如："myorg.internal"。
--skip-certificate-key-print	不要打印用于加密控制平面证书的密钥。
--skip-phases strings	要跳过的阶段列表
--skip-token-print	跳过打印 'kubeadm init' 生成的默认引导令牌。
--token string	这个令牌用于建立控制平面节点与工作节点间的双向通信。格式为 [a-z0-9]{6}.[a-z0-9]{16} - 示例：abcdef.0123456789abcdef
--token-ttl duration 默认值：24h0m0s	令牌被自动删除之前的持续时间（例如 1 s，2 m，3 h）。如果设置为 '0'，则令牌将永不过期
--upload-certs	将控制平面证书上传到 kubeadm-certs Secret。

初始化命令

 kubeadm init --apiserver-advertise-address 10.0.0.54 \
--apiserver-bind-port 6443 \
--control-plane-endpoint 10.0.0.53 \
--pod-network-cidr 172.16.0.0/16 \
--service-cidr 10.10.0.0/16 \
--image-repository registry.cn-hangzhou.aliyuncs.com/google_containers \
--kubernetes-version v1.20.5 \
--ignore-preflight-errors swap \
--service-dns-domain=shi.local \
#--upload-certs #该选项也可后续使用kubeadm init phase upload-certs --upload-certs获取

初始化完成输出信息如下：

 Your Kubernetes control-plane has initialized successfully!
 
To start using your cluster, you need to run the following as a regular user:
 
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
Alternatively, if you are the root user, you can run:
 
  export KUBECONFIG=/etc/kubernetes/admin.conf
 
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
 
You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:
 
  kubeadm join 10.0.0.53:6443 --token 4xe09u.grs65k149mu3ozja \
    --discovery-token-ca-cert-hash sha256:231cd8f418873ea811166d3876c9cb3765481451abbcf4e8b87155457999a36b \
    --control-plane
 
Then you can join any number of worker nodes by running the following on each as root:
 
kubeadm join 10.0.0.53:6443 --token 4xe09u.grs65k149mu3ozja \
    --discovery-token-ca-cert-hash sha256:231cd8f418873ea811166d3876c9cb3765481451abbcf4e8b87155457999a36b

配置kube-config文件

kube-config文件包含kube-apiserver地址及相关认证信息

 # 复制执行kubeadm --init初始化生成的命令
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl自动补全

bash

 source <(kubectl completion bash) # 在 bash 中设置当前 shell 的自动补全，要先安装 bash-completion 包。
echo "source <(kubectl completion bash)" >> ~/.bashrc # 在你的 bash shell 中永久地添加自动补全

ZSH

 source <(kubectl completion zsh)  # 在 zsh 中设置当前 shell 的自动补全
echo '[[ $commands[kubectl] ]] && source <(kubectl completion zsh)' >> ~/.zshrc # 在你的 zsh shell 中永久地添加自动补全

查看node信息

 [root@k8s-master1 ~]#kubectl get node
NAME          STATUS     ROLES                  AGE   VERSION
k8s-master1   NotReady   control-plane,master   54s   v1.20.5

部署网络组件flannel

进入https://github.com/flannel-io/flannel/tree/master/Documentation下载yml文件

注意：kube-flannel.yml文件参数需修改，yml文件中"10.244.0.0/16"和kubeadm --init初始化时--pod-network-cidr的值相同，参考kubernetes使用flannel网络插件服务状态显示CrashLoopBackOff

 # 若网络连接异常无法下载镜像时，选择开全局代理下载镜像
[root@k8s-master1 ~]#kubectl apply -f kube-flannel.yml
namespace/kube-flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
 
# 等待flannel初始化完成
[root@k8s-master1 ~]#kubectl get pod -A
NAMESPACE      NAME                                  READY   STATUS     RESTARTS   AGE
kube-flannel   kube-flannel-ds-bzgbv                 0/1     Init:1/2   0          16s
kube-system    coredns-54d67798b7-cb4mv              0/1     Pending    0          10m
kube-system    coredns-54d67798b7-lwmrz              0/1     Pending    0          10m
kube-system    etcd-k8s-master1                      1/1     Running    0          10m
kube-system    kube-apiserver-k8s-master1            1/1     Running    0          10m
kube-system    kube-controller-manager-k8s-master1   1/1     Running    0          10m
kube-system    kube-proxy-tpr65                      1/1     Running    0          10m
kube-system    kube-scheduler-k8s-master1            1/1     Running    0          10m
 
# 查看镜像
[root@k8s-master1 ~]#docker images
REPOSITORY                                                                    TAG                 IMAGE ID            CREATED             SIZE
rancher/mirrored-flannelcni-flannel                                           v0.19.1             252b2c3ee6c8        2 weeks ago         62.3MB
rancher/mirrored-flannelcni-flannel-cni-plugin                                v1.1.0              fcecffc7ad4a        3 months ago        8.09MB

查看master节点状态

 [root@k8s-master1 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE   VERSION
k8s-master1   Ready    control-plane,master   34m   v1.20.5

当前maser生成证书用于添加新控制节点

 [root@k8s-master1 ~]#kubeadm init phase upload-certs --upload-certs
I0825 10:08:25.373762  590886 version.go:254] remote version is much newer: v1.25.0; falling back to: stable-1.20
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
027da56ef05e18a7627135dca22fd571908eec8d1989df1dbbddd263782d2215

安装其他节点

安装docker、kubeadm、kubelet、kubectl，方法可见前面部分

 #! /bin/bash
apt update
 
# 安装依赖包
apt install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg \
    lsb-release \
    software-properties-common
 
# 安装GPG证书
curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
 
sudo add-apt-repository "deb [arch=$(dpkg --print-architecture)] http://mirrors.aliyun.com/docker-ce/linux/ubuntu \
  $(lsb_release -cs) stable"
 
apt update
# apt-cache madison docker-ce docker-ce-cli
apt -y install docker-ce=5:19.03.15~3-0~ubuntu-$(lsb_release -cs) \
docker-ce-cli=5:19.03.15~3-0~ubuntu-$(lsb_release -cs)
 
# 关闭防火墙
systemctl disable firewalld && systemctl stop firewalld
# 在/etc/hosts中添加IP、主机名
cat >> /etc/hosts <<EOF
`hostname -I|awk '{print $1}'` `hostname`
EOF
 
# 内核参数优化
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
 
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
 
sudo sysctl --system
 
# 设置docker的cgroup driver
# docker 默认的 cgroup driver 是 cgroupfs，可以通过 docker info 命令查看
# 如果用户没有在 KubeletConfiguration 下设置 cgroupDriver 字段，则 kubeadm 将默认为
cat <<EOF >/etc/docker/daemon.json
{
    "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
 
# 关闭swap
# 在/etc/fstab注释swap那一行
sed -ri 's/(^[^#]*swap)/#\1/' /etc/fstab
echo 'swapoff -a' >> /etc/profile
swapoff -a
 
# 添加本地harbor镜像仓库，HTTP连接方式
sed -ri 's/(^ExecStart.*$)/\1 --insecure-registry=10.0.0.60/' /lib/systemd/system/docker.service
systemctl daemon-reload && systemctl restart docker
 
# 设置国内阿里云镜像源
apt update && apt-get install -y apt-transport-https
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - 
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
deb https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial main
EOF
 
# 安装1.20.5版本kubeadm、kubelet、kubectl
apt update
apt install -y kubeadm=1.20.5-00 kubelet=1.20.5-00 kubectl=1.20.5-00
 
# 修改grub
sed -i '/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/c GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 cgroup_enable=memory swapaccount=1"' /etc/default/grub
update-grub
reboot

其他master节点加入集群

添加命令为k8s-master1执行kubeadm init初始化完成之后返回的kubeadm join信息+kubeadm init phase upload-certs --upload-certs生成的certificate-key信息

执行加入命令

   kubeadm join 10.0.0.53:6443 --token 4xe09u.grs65k149mu3ozja \
    --discovery-token-ca-cert-hash sha256:231cd8f418873ea811166d3876c9cb3765481451abbcf4e8b87155457999a36b \
    --control-plane --certificate-key 027da56ef05e18a7627135dca22fd571908eec8d1989df1dbbddd263782d2215

执行完成输出信息

 This node has joined the cluster and a new control plane instance was created:
 
* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.
 
To start administering your cluster from this node, you need to run the following as a regular user:
 
	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
Run 'kubectl get nodes' to see this node join the cluster.

配置kube-config文件

kube-config文件包含kube-apiserver地址及相关认证信息

 mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

kubectl自动补全

 source <(kubectl completion bash) # 在 bash 中设置当前 shell 的自动补全，要先安装 bash-completion 包。
echo "source <(kubectl completion bash)" >> ~/.bashrc # 在你的 bash shell 中永久地添加自动补全

node节点加入集群

执行加入命令

添加命令为k8s-master1执行kubeadm init初始化完成之后返回的kubeadm join信息

 kubeadm join 10.0.0.53:6443 --token 4xe09u.grs65k149mu3ozja \
    --discovery-token-ca-cert-hash sha256:231cd8f418873ea811166d3876c9cb3765481451abbcf4e8b87155457999a36b

执行完成输出信息

 [root@k8s-node3 ~]#kubeadm join 10.0.0.53:6443 --token 4xe09u.grs65k149mu3ozja \
>     --discovery-token-ca-cert-hash sha256:231cd8f418873ea811166d3876c9cb3765481451abbcf4e8b87155457999a36b
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
 
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
 
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

配置kube-config

将master端下的$HOME/.kube文件拷贝到node节点上

 [root@k8s-master1 ~]#scp -r /root/.kube 10.0.0.57:/root/

kubectl命令可正常使用

 [root@k8s-node1 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE   VERSION
k8s-master1   Ready    control-plane,master   10h   v1.20.5
k8s-master2   Ready    control-plane,master   26m   v1.20.5
k8s-master3   Ready    control-plane,master   16m   v1.20.5
k8s-node1     Ready    <none>                 15m   v1.20.5

配置kubectl自动补全

 source <(kubectl completion bash) # 在 bash 中设置当前 shell 的自动补全，要先安装 bash-completion 包。
echo "source <(kubectl completion bash)" >> ~/.bashrc # 在你的 bash shell 中永久地添加自动补全

验证集群所有节点状态

各node节点会自动加入到master节点，下载镜像并启动flannel，直到最终在master看到node处于Ready状态。

 [root@k8s-master1 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE    VERSION
k8s-master1   Ready    control-plane,master   10h    v1.20.5
k8s-master2   Ready    control-plane,master   12m    v1.20.5
k8s-master3   Ready    control-plane,master   2m5s   v1.20.5
k8s-node1     Ready    <none>                 115s   v1.20.5
k8s-node2     Ready    <none>                 54s    v1.20.5
k8s-node3     Ready    <none>                 49s    v1.20.5

验证当前证书状态

 [root@k8s-master1 ~]#kubectl get csr
NAME        AGE   SIGNERNAME                                    REQUESTOR                 CONDITION
csr-cxk7f   33m   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:4xe09u   Approved,Issued
csr-ffdw7   43m   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:4xe09u   Approved,Issued
csr-s6hdv   31m   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:4xe09u   Approved,Issued
csr-wnns2   32m   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:4xe09u   Approved,Issued
csr-zq264   33m   kubernetes.io/kube-apiserver-client-kubelet   system:bootstrap:4xe09u   Approved,Issued

创建容器并测试网络

创建容器

 [root@k8s-master1 ~]#kubectl run net-test1 --image=alpine sleep 360000
pod/net-test1 created
[root@k8s-master1 ~]#kubectl get pod -o wide
NAME        READY   STATUS    RESTARTS   AGE   IP           NODE        NOMINATED NODE   READINESS GATES
net-test1   1/1     Running   0          11s   172.16.5.2   k8s-node3   <none>           <none>

进入容器，测试内部网络

 [root@k8s-master1 ~]#kubectl exec -it net-test1 sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # ifconfig 
eth0      Link encap:Ethernet  HWaddr 16:B0:A1:C1:0A:7A  
          inet addr:172.16.5.2  Bcast:172.16.5.255  Mask:255.255.255.0
          UP BROADCAST RUNNING MULTICAST  MTU:1450  Metric:1
          RX packets:13 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:1178 (1.1 KiB)  TX bytes:42 (42.0 B)
 
lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:0 (0.0 B)  TX bytes:0 (0.0 B)
 
/ # ping -c1 -W1 10.0.0.54
PING 10.0.0.54 (10.0.0.54): 56 data bytes
64 bytes from 10.0.0.54: seq=0 ttl=63 time=0.289 ms
 
--- 10.0.0.54 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 0.289/0.289/0.289 ms
/ #

验证外部网络

 / # ping www.baidu.com
PING www.baidu.com (180.101.49.12): 56 data bytes
64 bytes from 180.101.49.12: seq=0 ttl=127 time=7.129 ms
64 bytes from 180.101.49.12: seq=1 ttl=127 time=6.786 ms
64 bytes from 180.101.49.12: seq=2 ttl=127 time=6.315 ms
64 bytes from 180.101.49.12: seq=3 ttl=127 time=6.840 ms
64 bytes from 180.101.49.12: seq=4 ttl=127 time=6.577 ms
^C
--- www.baidu.com ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max = 6.315/6.729/7.129 ms

查看haproxy监控状态页

部署dashboard

参考：https://github.com/kubernetes/dashboard

下载dashboard

地址：https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml

修改recommended.yaml中的Service部分

 kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  type: NodePort		#新增NodePort，可通过集群外部浏览器访问
  ports:
    - port: 443
      targetPort: 8443
      nodePort: 30002		#添加30002端口，可根据实际添加端口
  selector:
    k8s-app: kubernetes-dashboard

创建admin-user.yaml文件，进行鉴权

 apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
 
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

创建dashboard

 kubectl apply -f recommended.yaml admin-user.yaml

验证状态

查看service

 [root@k8s-master1 ~]#kubectl get svc -n kubernetes-dashboard 
NAME                        TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
dashboard-metrics-scraper   ClusterIP   10.10.3.238     <none>        8000/TCP        61m
kubernetes-dashboard        NodePort    10.10.224.229   <none>        443:30002/TCP   61m

查看serviceaccount

 [root@k8s-master1 ~]#kubectl get sa -n kubernetes-dashboard 
NAME                   SECRETS   AGE
admin-user             1         56m
default                1         63m
kubernetes-dashboard   1         63m
 
[root@k8s-master1 ~]#kubectl get secrets -n kubernetes-dashboard 
NAME                               TYPE                                  DATA   AGE
admin-user-token-tv25x             kubernetes.io/service-account-token   3      55m
default-token-679qb                kubernetes.io/service-account-token   3      61m
kubernetes-dashboard-certs         Opaque                                0      61m
kubernetes-dashboard-csrf          Opaque                                1      61m
kubernetes-dashboard-key-holder    Opaque                                2      61m
kubernetes-dashboard-token-94gzz   kubernetes.io/service-account-token   3      61m

查看token

 [root@k8s-master1 ~]#kubectl describe secrets admin-user-token-tv25x -n kubernetes-dashboard
Name:         admin-user-token-tv25x
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: 405d9bc5-3da8-45f1-8cc7-1a7f569ecdcd
 
Type:  kubernetes.io/service-account-token
 
Data
====
ca.crt:     1066 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6Imx0TjZKUTk5WnBKT1pMUHVXUnBLYzg1cG9nMTVMT183cG1wZ2E4MnBuRFUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLXR2MjV4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI0MDVkOWJjNS0zZGE4LTQ1ZjEtOGNjNy0xYTdmNTY5ZWNkY2QiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.ACazHf5aujd_Qm3-8WOzVO6HM0ndbqh8qwPaDKoK309HypIU__g2SIDvvTdztkfbpnLhU_WvBDuCJWJ_IxYihkJoFVW5vgKLFTmy0Mxl3WlBbfb2VmfxMluOEVJOslVlHgHYhrs2OX5X0B7uvzNgyuV5QuvWljA1rfzj_GphnRTCdA2uGZv1lgF_UGgwyl_s7pYwe6nhUqV843c1x9xv0-IDdQ0BS5MDeKA092oT6p6F74U7dRFbxeQ3lGi_3fw18tc_-uNPuQ0kiQzFHlQjZXSL0L4uzmTrZd0CnJrbSRzoROaqNHQQXuoECpoaAxyE79e3NwE_Pf8JwwDwAFOoZQ

访问任一集群节点30002端口

输入token，默认token时间是900s，15分钟，到期后会自动退出登陆。

进入首页，对集群进行管理

五、升级k8s集群1.21版本

官方说明：https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/

升级工作的基本流程如下：

升级主控制平面节点
升级其他控制平面节点
升级工作节点

升级准备

集群应使用静态的控制平面和 etcd Pod 或者外部 etcd。
务必备份所有重要组件，例如存储在数据库中应用层面的状态。 kubeadm upgrade 不会影响你的工作负载，只会涉及 Kubernetes 内部的组件，但备份终究是好的。
必须禁用交换分区。

注意事项

在对 kubelet 作次版本升版时需要腾空节点。对于控制面节点，其上可能运行着 CoreDNS Pods 或者其它非常重要的负载。
升级后，因为容器规约的哈希值已更改，所有容器都会被重新启动

  
[root@k8s-master1 ~]#kubeadm version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:08:27Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}
 
[root@k8s-master1 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE   VERSION
k8s-master1   Ready    control-plane,master   11h   v1.20.5
k8s-master2   Ready    control-plane,master   67m   v1.20.5
k8s-master3   Ready    control-plane,master   57m   v1.20.5
k8s-node1     Ready    <none>                 56m   v1.20.5
k8s-node2     Ready    <none>                 55m   v1.20.5
k8s-node3     Ready    <none>                 55m   v1.20.5
 
[root@k8s-master1 ~]#kubectl version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:10:43Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  },
  "serverVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:02:01Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}
 
[root@k8s-master1 ~]#kubelet --version
Kubernetes v1.20.5

确定要升级到哪个版本

使用操作系统的包管理器找到最新的稳定 1.21：

 [root@k8s-master1 ~]#apt update
[root@k8s-master1 ~]#apt-cache policy kubeadm
kubeadm:
  Installed: 1.20.5-00
  Candidate: 1.25.0-00
  Version table:
     1.25.0-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.24.4-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.24.3-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     ......
     1.22.1-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.22.0-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.21.14-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.21.13-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages
     1.21.12-00 500
        500 https://mirrors.aliyun.com/kubernetes/apt kubernetes-xenial/main amd64 Packages
        500 https://mirrors.tuna.tsinghua.edu.cn/kubernetes/apt kubernetes-xenial/main amd64 Packages

升级master节点

控制面节点上的升级过程应该每次处理一个节点。首先选择一个要先行升级的控制面节点。该节点上必须拥有 /etc/kubernetes/admin.conf 文件。

升级master1节点

对master节点进行组件升级，将管理端服务kube-controller-manager、kube-apiserver、kube-scheduler、kube-proxy等进行版本升级

升级kubeadm

升级 kubeadm：

 # 选择1.21最新稳定版本1.21.14-00
apt-get update && \
apt-get install -y --allow-change-held-packages kubeadm=1.21.14-00

验证下载操作正常，并且 kubeadm版本正确：

 # 查看kubeadm版本
[root@k8s-master1 ~]#kubectl version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "21",
    "gitVersion": "v1.21.14",
    "gitCommit": "0f77da5bd4809927e15d1658fb4aa8f13ad890a5",
    "gitTreeState": "clean",
    "buildDate": "2022-06-15T14:17:29Z",
    "goVersion": "go1.16.15",
    "compiler": "gc",
    "platform": "linux/amd64"
  },
  "serverVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:02:01Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}

验证升级计划：

 kubeadm upgrade plan

此命令检查你的集群是否可被升级，并取回你要升级的目标版本。命令也会显示一个包含组件配置版本状态的表格。

说明：kubeadm upgrade 也会自动对 kubeadm 在节点上所管理的证书执行续约操作。如果需要略过证书续约操作，可以使用标志 --certificate-renewal=false。更多的信息，可参阅证书管理指南。

说明：如果 kubeadm upgrade plan 给出任何需要手动升级的组件配置，用户必须通过 --config 命令行标志向 kubeadm upgrade apply 命令提供替代的配置文件。如果不这样做，kubeadm upgrade apply 会出错并退出，不再执行升级操作。

升级计划如下：

 Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT        TARGET
kubelet     5 x v1.20.5    v1.21.14
            1 x v1.21.14   v1.21.14
 
Upgrade to the latest stable version:
 
COMPONENT                 CURRENT    TARGET
kube-apiserver            v1.20.5    v1.21.14
kube-controller-manager   v1.20.5    v1.21.14
kube-scheduler            v1.20.5    v1.21.14
kube-proxy                v1.20.5    v1.21.14
CoreDNS                   1.7.0      v1.8.0
etcd                      3.4.13-0   3.4.13-0
 
You can now apply the upgrade by executing the following command:
 
	kubeadm upgrade apply v1.21.14		# 执行该命令升级

执行升级

选择要升级到的目标版本，运行合适的命令

 kubeadm upgrade apply v1.21.14 \
# --etcd-upgrade=false  #手动设置不升级etcd
 
# 升级成功信息如下
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.21.14". Enjoy!
 
[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.

手动升级 CNI 驱动插件

容器网络接口（CNI）驱动应该提供了程序自身的升级说明。参阅插件页面查找 CNI 驱动，并查看是否需要其他升级步骤。

如果 CNI 驱动作为 DaemonSet 运行，则在其他控制平面节点上不需要此步骤。

腾空节点

通过将节点标记为不可调度并腾空节点为节点作升级准备

 # 标记k8s-master1 节点为不可调度
[root@k8s-master1 ~]#kubectl cordon k8s-master1
node/k8s-master1 cordoned
[root@k8s-master1 ~]#kubectl get node
NAME          STATUS                     ROLES                  AGE     VERSION
k8s-master1   Ready,SchedulingDisabled   control-plane,master   15h     v1.20.5
k8s-master2   Ready                      control-plane,master   4h29m   v1.20.5
k8s-master3   Ready                      control-plane,master   4h19m   v1.20.5
 
# 对k8s-master1 节点进行清空操作，为节点维护做准备
[root@k8s-master1 ~]#kubectl drain k8s-master1 --ignore-daemonsets --force=true
node/k8s-master1 already cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-flannel/kube-flannel-ds-n8b9v, kube-system/kube-proxy-tpr65
evicting pod kube-system/coredns-54d67798b7-lwmrz
evicting pod kube-system/coredns-54d67798b7-cb4mv
pod/coredns-54d67798b7-lwmrz evicted
pod/coredns-54d67798b7-cb4mv evicted
node/k8s-master1 evicted

说明

 kubectl cordon my-node                                                # 标记 my-node 节点为不可调度
kubectl drain my-node                                                 # 对 my-node 节点进行清空操作，为节点维护做准备
kubectl uncordon my-node                                              # 标记 my-node 节点为可以调度

升级kubelet、kubectl

 # 选择1.21最新稳定版本1.21.14-00
apt-get update && \
apt-get install -y --allow-change-held-packages kubelet=1.21.14-00 kubectl=1.21.14-00

验证下载操作正常，并且 kubeadm、kubectl、kubelet版本正确

 # 查看kubectl版本
[root@k8s-master1 ~]#kubectl version -o json
{
  "clientVersion": {
    "major": "1",
    "minor": "21",
    "gitVersion": "v1.21.5",
    "gitCommit": "aea7bbadd2fc0cd689de94a54e5b7b758869d691",
    "gitTreeState": "clean",
    "buildDate": "2021-09-15T21:10:45Z",
    "goVersion": "go1.16.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  },
  "serverVersion": {
    "major": "1",
    "minor": "20",
    "gitVersion": "v1.20.5",
    "gitCommit": "6b1d87acf3c8253c123756b9e61dac642678305f",
    "gitTreeState": "clean",
    "buildDate": "2021-03-18T01:02:01Z",
    "goVersion": "go1.15.8",
    "compiler": "gc",
    "platform": "linux/amd64"
  }
}
# 查看kubelet版本
[root@k8s-master1 ~]#kubelet --version
Kubernetes v1.21.14

重启kubelet

 sudo systemctl daemon-reload
sudo systemctl restart kubelet

取消节点保护

 # 将节点标记为可调度，让节点重新上线
kubectl uncordon k8s-master1

验证状态

 [root@k8s-master1 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE     VERSION
k8s-master1   Ready    control-plane,master   17h     v1.21.14
k8s-master2   Ready    control-plane,master   7h26m   v1.20.5
k8s-master3   Ready    control-plane,master   7h16m   v1.20.5
k8s-node1     Ready    <none>                 7h16m   v1.20.5
k8s-node2     Ready    <none>                 7h15m   v1.20.5
k8s-node3     Ready    <none>                 7h14m   v1.20.5

升级其他master节点

升级kubeadm

 # 下载kubeadm
apt-get update && \
apt-get install -y --allow-change-held-packages kubeadm=1.21.14-00
 
# 执行节点升级，使用kubeadm upgrade node，不需要执行 kubeadm upgrade plan 和更新 CNI 驱动插件的操作
sudo kubeadm upgrade node

升级成功信息如下

 [upgrade/staticpods] Component "kube-scheduler" upgraded successfully!
[upgrade] The control plane instance for this node was successfully updated!
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[upgrade] The configuration for this node was successfully updated!
[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.

腾空节点

 # master2
kubectl cordon k8s-master2
kubectl drain k8s-master2 --ignore-daemonsets --force=true
 
# master3
kubectl cordon k8s-master3
kubectl drain k8s-master3 --ignore-daemonsets --force=true

升级kubelet、kubectl

 apt-get update && \
apt-get install -y --allow-change-held-packages kubelet=1.21.14-00 kubectl=1.21.14-00

重启kubelet

 sudo systemctl daemon-reload
sudo systemctl restart kubelet

取消节点保护

 # master2
kubectl uncordon k8s-master2
 
# master3
kubectl uncordon k8s-master3

验证状态

 [root@k8s-master3 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE     VERSION
k8s-master1   Ready    control-plane,master   18h     v1.21.14
k8s-master2   Ready    control-plane,master   7h54m   v1.21.14
k8s-master3   Ready    control-plane,master   7h44m   v1.21.14
k8s-node1     Ready    <none>                 7h44m   v1.20.5
k8s-node2     Ready    <none>                 7h43m   v1.20.5
k8s-node3     Ready    <none>                 7h43m   v1.20.5

升级node节点

升级kubeadm

 # 下载kubeadm
apt-get update && \
apt-get install -y --allow-change-held-packages kubeadm=1.21.14-00
 
# 执行节点升级，对于工作节点，该命令会升级本地的 kubelet 配置
sudo kubeadm upgrade node

腾空节点

 kubectl cordon `hostname`
kubectl drain `hostname` --ignore-daemonsets --force=true

升级kubelet、kubectl

 # 升级kubelet、kubectl
apt-get update && \
apt-get install -y --allow-change-held-packages kubelet=1.21.14-00 kubectl=1.21.14-00
 
# 重启kubelet
sudo systemctl daemon-reload
sudo systemctl restart kubelet

取消节点保护

 kubectl uncordon `hostname`

验证集群状态

所有节点STATUS 应显示为 Ready 状态，并且版本号已经被更新

 [root@k8s-node3 ~]#kubectl get node
NAME          STATUS   ROLES                  AGE     VERSION
k8s-master1   Ready    control-plane,master   18h     v1.21.14
k8s-master2   Ready    control-plane,master   8h      v1.21.14
k8s-master3   Ready    control-plane,master   7h57m   v1.21.14
k8s-node1     Ready    <none>                 7h57m   v1.21.14
k8s-node2     Ready    <none>                 7h56m   v1.21.14
k8s-node3     Ready    <none>                 7h56m   v1.21.14

六、通过deployment.yaml 部署pod，镜像为nginx，3副本，并编辑index内容，分别显示各自pod的IP

参考：https://kubernetes.io/zh-cn/docs/concepts/workloads/controllers/deployment/

准备yaml文件

 apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.14.2
        ports:
        - containerPort: 80

创建deployment

 [root@k8s-master1 ~]#kubectl apply -f deploy-nginx.yml 
deployment.apps/nginx-deployment created
 
# 查看deployment
[root@k8s-master1 ~]#kubectl get deploy nginx-deployment 
NAME               READY   UP-TO-DATE   AVAILABLE   AGE
nginx-deployment   3/3     3            3           28s
 
# 查看pod
[root@k8s-master1 ~]#kubectl get pod -o wide
NAME                                READY   STATUS    RESTARTS   AGE    IP           NODE        NOMINATED NODE   READINESS GATES
nginx-deployment-66b6c48dd5-b8wxl   1/1     Running   0          105s   172.16.4.4   k8s-node2   <none>           <none>
nginx-deployment-66b6c48dd5-lfn7g   1/1     Running   0          105s   172.16.3.4   k8s-node1   <none>           <none>
nginx-deployment-66b6c48dd5-nv2md   1/1     Running   0          105s   172.16.5.5   k8s-node3   <none>           <none>

进入pod容器，编辑index内容

 # 进入ppod
[root@k8s-master1 ~]#kubectl exec -it nginx-deployment-66b6c48dd5-b8wxl bash
 
# 修改index内容
root@nginx-deployment-66b6c48dd5-b8wxl:/# hostname -i > /usr/share/nginx/html/index.html

验证

显示pod详细信息

 [root@k8s-master1 ~]#kubectl get pod -o wide
NAME                                READY   STATUS    RESTARTS   AGE   IP           NODE        NOMINATED NODE   READINESS GATES
nginx-deployment-66b6c48dd5-b8wxl   1/1     Running   0          36m   172.16.4.4   k8s-node2   <none>           <none>
nginx-deployment-66b6c48dd5-lfn7g   1/1     Running   0          36m   172.16.3.4   k8s-node1   <none>           <none>
nginx-deployment-66b6c48dd5-nv2md   1/1     Running   0          36m   172.16.5.5   k8s-node3   <none>           <none>

显示各自pod的IP

 [root@k8s-master1 ~]#curl 172.16.4.4
172.16.4.4
 
[root@k8s-master1 ~]#curl 172.16.3.4
172.16.3.4
 
[root@k8s-master1 ~]#curl 172.16.5.5
172.16.5.5

七、创建svc，用来访问已知的nginx pod

创建ClusterPort类型service

只能集群内部访问

准备service yaml文件

 apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  selector:
    app: nginx
  ports:
    - protocol: TCP
      port: 8080		# service端口
      targetPort: 80		# pod端口

创建service

 # 创建service
[root@k8s-master1 ~]#kubectl apply -f nginx-service.yaml 
service/nginx-service created
 
# 查看service
[root@k8s-master1 ~]#kubectl get svc -o wide
NAME            TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)    AGE   SELECTOR
kubernetes      ClusterIP   10.10.0.1       <none>        443/TCP    20h   <none>
nginx-service   ClusterIP   10.10.188.189   <none>        8080/TCP   13m   app=nginx
 
# 查看endpoints
[root@k8s-master1 ~]#kubectl get ep
NAME            ENDPOINTS                                      AGE
kubernetes      10.0.0.54:6443,10.0.0.55:6443,10.0.0.56:6443   20h
nginx-service   172.16.3.4:80,172.16.4.4:80,172.16.5.5:80      14m

验证，集群内部访问，随机调度至后端pod

 [root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.4.4
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.4.4
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.3.4
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.5.5
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.5.5
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.4.4
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.4.4
[root@k8s-master1 ~]#curl 10.10.188.189:8080
172.16.4.4

创建NodePort类型Service

集群外部可访问

准备service yaml文件

 apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  selector:
    app: nginx
  type: NodePort
  ports:
    - protocol: TCP
      port: 8080		# service暴露端口
      targetPort: 80		# pod暴露端口
      nodePort: 30080		# node暴露端口，端口范围30000-32767

创建service

 # 创建service
[root@k8s-master1 ~]#kubectl apply -f nginx-service-node.yaml
service/nginx-service created
 
# 查看service
[root@k8s-master1 ~]#kubectl get svc -o wide
NAME            TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE   SELECTOR
kubernetes      ClusterIP   10.10.0.1      <none>        443/TCP          21h   <none>
nginx-service   NodePort    10.10.81.129   <none>        8080:30080/TCP   92s   app=nginx
 
# 查看endpoints
[root@k8s-master1 ~]#kubectl get ep
NAME            ENDPOINTS                                      AGE
kubernetes      10.0.0.54:6443,10.0.0.55:6443,10.0.0.56:6443   21h
nginx-service   172.16.3.4:80,172.16.4.4:80,172.16.5.5:80      116s

验证，集群外部访问，随机调度至后端pod

 [root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.3.4
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.3.4
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.4.4
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.5.5
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.3.4
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.5.5
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.3.4
[root@haproxy1 ~]#curl 10.0.0.54:30080
172.16.4.4

浏览器访问

posted @ 2022-08-25 20:58 areke 阅读(900) 评论(0) 编辑收藏举报

刷新页面返回顶部

登录后才能查看或发表评论，立即登录或者逛逛博客园首页

相关博文：

· kubernetes（二）

· kubernetes（一）

· 使用kubernetes 官网工具kubeadm部署kubernetes(使用阿里云镜像)

· Kubernetes 集群安装

· kubernetes 集群安装，离线版

阅读排行：
· 全程不用写代码，我用AI程序员写了一个飞机大战
· MongoDB 8.0这个新功能碉堡了，比商业数据库还牛
· 记一次.NET内存居高不下排查解决与启示
· 白话解读 Dapr 1.15：你的「微服务管家」又秀新绝活了
· DeepSeek 开源周回顾「GitHub 热点速览」

公告

昵称： areke
园龄： 3年3个月
粉丝： 5
关注： 0

+加关注

2025年3月

日

一

二

三

四

五

六

	#! /bin/bash
	apt update

	apt install \
	ca-certificates \
	curl \
	gnupg \
	lsb-release

	mkdir -p /etc/apt/keyrings
	curl -fsSL https://download.docker.com/linux/ubuntu/gpg \| gpg --dearmor -o /etc/apt/keyrings/docker.gpg

	echo \
	"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/ubuntu \
	$(lsb_release -cs) stable" \| sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

	apt update
	apt -y install docker-ce docker-ce-cli containerd.io docker-compose-plugin

	sed -i '/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/c GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0 cgroup_enable=memory swapaccount=1"' /etc/default/grub
	update-grub
	reboot

	[root@jumpserver ~]#docker --version
	Docker version 20.10.17, build 100c701

	docker pull mysql:5.6
	docker pull redis:6.0
	docker pull jumpserver/jms_all:1.5.9

	[root@jumpserver ~]#docker images
	REPOSITORY TAG IMAGE ID CREATED SIZE
	redis 6.0 6487e6a1b120 2 weeks ago 112MB
	mysql 5.6 dd3b2a5dcb48 8 months ago 303MB
	jumpserver/jms_all 1.5.9 6573e4b624c9 2 years ago 1.48GB

	mkdir -p /data/mysql
	mkdir -p /data/etc/mysql/conf.d
	mkdir -p /data/etc/mysql/mysql.conf.d

	[root@jumpserver ~]#cat /data/etc/mysql/conf.d/mysql.cnf
	[mysql]
	default-character-set=utf8

	[root@jumpserver ~]#cat /data/etc/mysql/mysql.conf.d/mysqld.cnf
	[mysqld]
	pid-file = /var/run/mysqld/mysqld.pid
	socket = /var/run/mysqld/mysqld.sock
	datadir = /var/lib/mysql
	log-error = /var/log/mysql/error.log
	symbolic-links=0
	character-set-server=utf8

	[root@jumpserver ~]#tree /data
	/data
	├── etc
	│ └── mysql
	│ ├── conf.d
	│ │ └── mysql.cnf
	│ └── mysql.conf.d
	│ └── mysqld.cnf
	└── mysql

	5 directories, 2 files

	docker run -it -d -p 3306:3306 -v /data/etc/mysql/conf.d/mysql.cnf:/etc/mysql/conf.d/mysql.cnf \
	-v /data/etc/mysql/mysql.conf.d/mysqld.cnf:/etc/mysql/mysql.conf.d/mysqld.cnf \
	-v /data/mysql:/var/lib/mysql -e MYSQL_ROOT_PASSWORD='123456' mysql:5.6

	# 宿主机连接，先安装apt install mysql-client
	[root@jumpserver mysql]#mysql -uroot -p123456 -h10.0.0.12
	mysql: [Warning] Using a password on the command line interface can be insecure.
	Welcome to the MySQL monitor. Commands end with ; or \g.
	Your MySQL connection id is 2
	Server version: 5.6.51 MySQL Community Server (GPL)

	Copyright (c) 2000, 2022, Oracle and/or its affiliates.

	Oracle is a registered trademark of Oracle Corporation and/or its
	affiliates. Other names may be trademarks of their respective
	owners.

	Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

	mysql> show databases;
	+--------------------+
	\| Database \|
	+--------------------+
	\| information_schema \|
	\| mysql \|
	\| performance_schema \|
	+--------------------+
	3 rows in set (0.01 sec)

	mysql> show variables like "%character%";
	+--------------------------+----------------------------+
	\| Variable_name \| Value \|
	+--------------------------+----------------------------+
	\| character_set_client \| utf8 \|
	\| character_set_connection \| utf8 \|
	\| character_set_database \| utf8 \|
	\| character_set_filesystem \| binary \|
	\| character_set_results \| utf8 \|
	\| character_set_server \| utf8 \|
	\| character_set_system \| utf8 \|
	\| character_sets_dir \| /usr/share/mysql/charsets/ \|
	+--------------------------+----------------------------+
	8 rows in set (0.01 sec)

	mysql> show variables like "%collation%";
	+----------------------+-----------------+
	\| Variable_name \| Value \|
	+----------------------+-----------------+
	\| collation_connection \| utf8_general_ci \|
	\| collation_database \| utf8_general_ci \|
	\| collation_server \| utf8_general_ci \|
	+----------------------+-----------------+
	3 rows in set (0.00 sec)

	[root@jumpserver data]#mysql -uroot -p123456 -h10.0.0.12

	mysql> create database jumpserver default charset 'utf8';
	Query OK, 1 row affected (0.00 sec)

	mysql> grant all on jumpserver.* to 'jumpserver'@'%' identified by 'jp123456';
	Query OK, 0 rows affected (0.00 sec)

	[root@jumpserver mysql.conf.d]#mysql -ujumpserver -pjp123456 -h10.0.0.12
	mysql: [Warning] Using a password on the command line interface can be insecure.
	Welcome to the MySQL monitor. Commands end with ; or \g.
	Your MySQL connection id is 2
	Server version: 5.6.51 MySQL Community Server (GPL)

	Copyright (c) 2000, 2022, Oracle and/or its affiliates.

	Oracle is a registered trademark of Oracle Corporation and/or its
	affiliates. Other names may be trademarks of their respective
	owners.

	Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

	mysql> show databases;
	+--------------------+
	\| Database \|
	+--------------------+
	\| information_schema \|
	\| jumpserver \|
	+--------------------+
	2 rows in set (0.00 sec)

	[root@jumpserver mysql]#docker run -d -it -p 6379:6379 redis:6.0
	331baf1779e94bc110f4d15fad2e0bce743ff17c051862216bc1d629ba47177c

	[root@jumpserver mysql]#redis-cli -h 10.0.0.12
	10.0.0.12:6379> info
	# Server
	redis_version:6.0.16
	redis_git_sha1:00000000
	redis_git_dirty:0
	redis_build_id:b9d515b0d5e963de
	redis_mode:standalone
	os:Linux 5.4.0-122-generic x86_64
	arch_bits:64

	# 随机密钥
	cat /dev/urandom \|tr -dc A-Za-z0-9\|head -c 50 >> /data/secret_key
	j3GFsi5Qwcn5Jj4rugfwvdBc56B6QMkyBuncwRP72XviP7mxTU

	# token
	cat /dev/urandom \|tr -dc A-Za-z0-9\|head -c 16 >> bootstrap_token
	kiBEmIVpWrFrlY3q

	docker run -d --name jms_all \
	-v /data/jumpserver:/opt/jumpserver/data/media \
	-p 80:80 \
	-p 2222:2222 \
	-e SECRET_KEY=j3GFsi5Qwcn5Jj4rugfwvdBc56B6QMkyBuncwRP72XviP7mxTU \
	-e BOOTSTRAP_TOKEN=kiBEmIVpWrFrlY3q \
	-e DB_HOST=10.0.0.12 \
	-e DB_PORT=3306 \
	-e DB_USER='jumpserver' \
	-e DB_PASSWORD='jp123456' \
	-e DB_NAME=jumpserver \
	-e REDIS_HOST=10.0.0.12 \
	-e REDIS_PORT=6379 \
	-e REDIS_PASSWORD= \
	jumpserver/jms_all:1.5.9

	Starting guacd: guacd[91]: INFO: Guacamole proxy daemon (guacd) version 1.0.0 started
	SUCCESS
	Tomcat started.
	Jumpserver ALL 1.5.9
	官网 http://www.jumpserver.org
	文档 http://docs.jumpserver.org
	有问题请参考 http://docs.jumpserver.org/zh/docs/faq.html

	进入容器命令 docker exec -it jms_all /bin/bash

	[root@jumpserver ~]#mysql -ujumpserver -pjp123456 -h10.0.0.12
	mysql: [Warning] Using a password on the command line interface can be insecure.
	Welcome to the MySQL monitor. Commands end with ; or \g.
	Your MySQL connection id is 24
	Server version: 5.6.51 MySQL Community Server (GPL)

	Copyright (c) 2000, 2022, Oracle and/or its affiliates.

	Oracle is a registered trademark of Oracle Corporation and/or its
	affiliates. Other names may be trademarks of their respective
	owners.

	Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.

	mysql> show databases;
	+--------------------+
	\| Database \|
	+--------------------+
	\| information_schema \|
	\| jumpserver \|
	+--------------------+
	2 rows in set (0.06 sec)

	mysql> use jumpserver;
	Reading table information for completion of table and column names
	You can turn off this feature to get a quicker startup with -A

	Database changed
	mysql> show tables;
	+----------------------------------------------+
	\| Tables_in_jumpserver \|
	+----------------------------------------------+
	\| applications_databaseapp \|
	\| applications_remoteapp \|
	\| assets_adminuser \|
	\| assets_asset \|
	\| assets_asset_labels \|
	\| assets_asset_nodes \|
	\| assets_assetgroup \|
	\| assets_authbook \|
	\| assets_cluster \|

	新建过滤器名称

	<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210807192-729271614.png" width="50%" height="50%" />


	点击过滤器名称

	<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210814463-1057751346.png" width="50%" height="50%" />


	创建规则

	<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210843768-62335829.png" width="50%" height="50%" />


	配置过滤器具体规则

	<img src="https://img2022.cnblogs.com/blog/2621461/202208/2621461-20220825210835210-274524939.png" width="50%" height="50%" />

areke

k8s、jumpserver

一、安装jumpserver

安装docker

下载镜像

部署mysql

部署redis

部署jumpserver

登录web界面

二、配置用户tom，并添加一个相应的服务器资源

创建规则

登录测试

三、使用命令过滤器禁止tom用户在服务器上使用 reboot、init、rm -rf、poweroff 这几个命令

四、部署k8s指定版本1.20高可用集群

安装高可用反向代理

安装keepalived

节点1（10.0.0.51）

节点2（10.0.0.52）

安装haproxy

节点1（10.0.0.51）

节点2（10.0.0.52）

查看haproxy监控状态页

安装harbor镜像仓库

所有节点安装kubeadm组件

安装docker

系统优化

设置国内阿里云镜像源

安装kubelet、kubeadm、kubectl

kubeadm init集群初始化

高可用master初始化

配置kube-config文件

部署网络组件flannel

当前maser生成证书用于添加新控制节点

安装其他节点

安装docker、kubeadm、kubelet、kubectl，方法可见前面部分

其他master节点加入集群

执行加入命令

配置kube-config文件

kubectl自动补全

node节点加入集群

执行加入命令

配置kube-config

配置kubectl自动补全

验证集群所有节点状态

验证当前证书状态

创建容器并测试网络

查看haproxy监控状态页

部署dashboard

下载dashboard

创建dashboard

验证状态

访问任一集群节点30002端口

五、升级k8s集群1.21版本

升级准备

确定要升级到哪个版本

升级master节点

升级master1节点

升级kubeadm

腾空节点

升级kubelet、kubectl

取消节点保护

升级其他master节点

升级kubeadm

腾空节点

升级kubelet、kubectl

取消节点保护

升级node节点

升级kubeadm

腾空节点

升级kubelet、kubectl

取消节点保护

验证集群状态

六、通过deployment.yaml 部署pod，镜像为nginx，3副本，并编辑index内容，分别显示各自pod的IP

七、创建svc，用来访问已知的nginx pod

创建ClusterPort类型service

创建NodePort类型Service

公告

搜索

常用链接

我的标签

	root@haproxy1 keepalived]#vim /etc/keepalived/keepalived.conf
	! Configuration File for keepalived

	global_defs {
	notification_email {
	acassen@firewall.loc
	failover@firewall.loc
	sysadmin@firewall.loc
	}
	notification_email_from Alexandre.Cassen@firewall.loc
	smtp_server 192.168.200.1
	smtp_connect_timeout 30
	router_id 10.0.0.51
	vrrp_skip_check_adv_addr
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	vrrp_mcast_group4 224.0.0.18
	}

	# 脚本需放在调用之前，先定义好
	vrrp_script chk_haproxy {
	script "/etc/keepalived/chk_haproxy.sh"
	interval 1
	timeout 2
	weight -30
	fall 3
	rise 5
	}

	vrrp_instance haproxy {
	state MASTER
	interface eth0
	virtual_router_id 51
	priority 100
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	10.0.0.53/24 dev eth0 label eth0:1
	}

	track_script {
	chk_haproxy
	}

	}

	[root@haproxy1 keepalived#vim /etc/keepalived/chk_haproxy.sh
	#!/bin/bash
	/usr/bin/killall -0 haproxy

	#添加执行权限
	[root@haproxy1 keepalived# chmod a+x /etc/keepalived/chk_haproxy.sh


	[root@haproxy1 keepalived]#ifconfig -a
	eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.51 netmask 255.255.255.0 broadcast 10.0.0.255
	inet6 fe80::20c:29ff:fef9:b4ed prefixlen 64 scopeid 0x20<link>
	ether 00:0c:29:f9:b4:ed txqueuelen 1000 (Ethernet)
	RX packets 20377 bytes 23085737 (23.0 MB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 11683 bytes 1239148 (1.2 MB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.53 netmask 255.255.255.0 broadcast 0.0.0.0
	ether 00:0c:29:f9:b4:ed txqueuelen 1000 (Ethernet)

	lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 127.0.0.1 netmask 255.0.0.0
	inet6 ::1 prefixlen 128 scopeid 0x10<host>
	loop txqueuelen 1000 (Local Loopback)
	RX packets 0 bytes 0 (0.0 B)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 0 bytes 0 (0.0 B)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	[root@haproxy2 ~]#vim /etc/keepalived/keepalived.conf

	! Configuration File for keepalived

	global_defs {
	notification_email {
	acassen@firewall.loc
	failover@firewall.loc
	sysadmin@firewall.loc
	}
	notification_email_from Alexandre.Cassen@firewall.loc
	smtp_server 192.168.200.1
	smtp_connect_timeout 30
	router_id 10.0.0.52 #与节点1不同，为本机IP
	vrrp_skip_check_adv_addr
	vrrp_garp_interval 0
	vrrp_gna_interval 0
	vrrp_mcast_group4 224.0.0.18
	}

	vrrp_instance haproxy {
	state BACKUP #BACKUP
	interface eth0
	virtual_router_id 51
	priority 80 #优先级80
	advert_int 1
	authentication {
	auth_type PASS
	auth_pass 1111
	}
	virtual_ipaddress {
	10.0.0.10/24 dev eth0 label eth0:1
	}
	}

	[root@haproxy2 ~]#vim /etc/keepalived/chk_haproxy.sh
	#!/bin/bash
	/usr/bin/killall -0 haproxy

	#添加执行权限
	[root@haproxy2 ~]#chmod a+x /etc/keepalived/chk_haproxy.sh

	[root@haproxy2 ~]#ifconfig
	eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.52 netmask 255.255.255.0 broadcast 10.0.0.255
	inet6 fe80::20c:29ff:fe3d:976d prefixlen 64 scopeid 0x20<link>
	ether 00:0c:29:3d:97:6d txqueuelen 1000 (Ethernet)
	RX packets 3371 bytes 3424676 (3.4 MB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 2136 bytes 253767 (253.7 KB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.53 netmask 255.255.255.0 broadcast 0.0.0.0
	ether 00:0c:29:3d:97:6d txqueuelen 1000 (Ethernet)

	lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 127.0.0.1 netmask 255.0.0.0
	inet6 ::1 prefixlen 128 scopeid 0x10<host>
	loop txqueuelen 1000 (Local Loopback)
	RX packets 0 bytes 0 (0.0 B)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 0 bytes 0 (0.0 B)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	[root@haproxy1 ~]#vim /etc/haproxy/haproxy.cfg
	....
	#末尾添加如下配置
	listen stats
	mode http
	bind 0.0.0.0:9999
	stats enable
	log global
	stats uri /haproxy-status
	stats auth haadmin:123456

	listen k8s-6443
	bind 10.0.0.53:6443
	mode tcp
	balance roundrobin
	server k8s-master1 10.0.0.54:6443 check inter 3s fall 3 rise 5
	server k8s-master2 10.0.0.55:6443 check inter 3s fall 3 rise 5
	server k8s-master3 10.0.0.56:6443 check inter 3s fall 3 rise 5

	#限制响应级别：arp_ignore
	#0：默认值，表示可使用本地任意接口上配置的任意地址进行响应
	#1：仅在请求的目标IP配置在本地主机的接收到请求报文的接口上时，才给予响应
	echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
	echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore

	#限制通告级别：arp_announce
	#0：默认值，把本机所有接口的所有信息向每个接口的网络进行通告
	#1：尽量避免将接口信息向非直接连接网络进行通告
	#2：必须避免将接口信息向非本网络进行通告
	echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
	echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce

	echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf #开启后VIP不在本地，haproxy也可绑定该地址
	echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf #开启ipv4路由转发功能

	[root@haproxy1 ~]#ifconfig
	eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.51 netmask 255.255.255.0 broadcast 10.0.0.255
	inet6 fe80::20c:29ff:fef9:b4ed prefixlen 64 scopeid 0x20<link>
	ether 00:0c:29:f9:b4:ed txqueuelen 1000 (Ethernet)
	RX packets 24898 bytes 25024044 (25.0 MB)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 15962 bytes 1611934 (1.6 MB)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	eth0:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
	inet 10.0.0.53 netmask 255.255.255.0 broadcast 0.0.0.0
	ether 00:0c:29:f9:b4:ed txqueuelen 1000 (Ethernet)

	lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
	inet 127.0.0.1 netmask 255.0.0.0
	inet6 ::1 prefixlen 128 scopeid 0x10<host>
	loop txqueuelen 1000 (Local Loopback)
	RX packets 0 bytes 0 (0.0 B)
	RX errors 0 dropped 0 overruns 0 frame 0
	TX packets 0 bytes 0 (0.0 B)
	TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

	[root@haproxy1 ~]#netstat -ntlp
	Active Internet connections (only servers)
	Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
	tcp 0 0 10.0.0.53:6443 0.0.0.0:* LISTEN 7783/haproxy
	tcp 0 0 0.0.0.0:9999 0.0.0.0:* LISTEN 7783/haproxy
	tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/init
	tcp 0 0 0.0.0.0:46961 0.0.0.0:* LISTEN 824/rpc.mountd
	tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 822/systemd-resolve
	tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 915/sshd: /usr/sbin
	tcp 0 0 127.0.0.1:6010 0.0.0.0:* LISTEN 1493/sshd: root@pts
	tcp 0 0 0.0.0.0:2049 0.0.0.0:* LISTEN -
	tcp 0 0 0.0.0.0:35845 0.0.0.0:* LISTEN 824/rpc.mountd
	tcp 0 0 0.0.0.0:37381 0.0.0.0:* LISTEN 824/rpc.mountd
	tcp 0 0 0.0.0.0:44905 0.0.0.0:* LISTEN -
	tcp6 0 0 :::37675 :::* LISTEN -
	tcp6 0 0 :::111 :::* LISTEN 1/init
	tcp6 0 0 :::22 :::* LISTEN 915/sshd: /usr/sbin
	tcp6 0 0 ::1:6010 :::* LISTEN 1493/sshd: root@pts
	tcp6 0 0 :::47389 :::* LISTEN 824/rpc.mountd
	tcp6 0 0 :::42335 :::* LISTEN 824/rpc.mountd
	tcp6 0 0 :::54751 :::* LISTEN 824/rpc.mountd
	tcp6 0 0 :::2049 :::* LISTEN -