K8S 高可用集群

 

一、Kubernetes平台环境规划

 

 

二、部署NGINX负载均衡（例如stream模块：stream是NGINX新加支持四层负载）

Nginx01和Nginx02 10.192.27.111 10.192.27.112 

[root@nginx01 ~]# cat > /etc/yum.repos.d/nginx.repo << EOF
> [nginx]
> name=nginx repo
> baseurl=http://nginx.org/packages/centos/7/$basearch/
> gpgcheck=0
> EOF
[root@nginx01 ~]# cat /etc/yum.repos.d/nginx.repo 
[nginx]
name=nginx repo
baseurl=http://nginx.org/packages/centos/7/$basearch/  #baseurl=http://nginx.org/packages/centos/7/$basearch/
gpgcheck=0

[root@nginx01 ~]# yum -y install nginx
[root@nginx01 ~]# cat /etc/nginx/nginx.conf   #修改后的配置文件

 

 

user  nginx;
worker_processes  1;

error_log  /var/log/nginx/error.log warn;
pid        /var/run/nginx.pid;


events {
    worker_connections  1024;
}

stream { 

   log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log  /var/log/nginx/k8s-access.log  main;

    upstream k8s-apiserver {
        server 10.192.27.100:6443;  #后端master节点
        server 10.192.27.114:6443;
    }
    server {
                listen 6443;
                proxy_pass k8s-apiserver;
    }
    }



http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    keepalive_timeout  65;

    #gzip  on;

    include /etc/nginx/conf.d/*.conf;
}

 

 

启动服务

[root@nginx01 ~]# systemctl start nginx  #要关闭防火墙
[root@nginx01 ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.

 

 

查看端口

[root@nginx01 ~]# netstat -anput | grep 6443
tcp        0      0 0.0.0.0:6443            0.0.0.0:*               LISTEN      7166/nginx: master 

 

 

拷贝配置

####################NGINX01
[root@nginx01 ~]#  echo "nginx master" > /usr/share/nginx/html/index.html  
[root@nginx01 ~]# scp /etc/nginx/nginx.conf root@10.192.27.112:/etc/nginx/  #配置文件拷贝至master02
[root@nginx01 ~]# 
####################NGINX02
[root@nginx02 ~]# systemctl start nginx
[root@nginx02 ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.
[root@nginx02 ~]# echo "nginx backup" > /usr/share/nginx/html/index.html 
[root@nginx02 ~]# 

 

 

 

 

 

 

三、部署keepalived

1. Nginx01配置

[root@nginx01 ~]# yum -y install keepalived
[root@nginx01 ~]# vi /etc/keepalived/keepalived.conf   keepalived配置文件
! Configuration File for keepalived 
 
global_defs { 
   # 接收邮件地址 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   # 邮件发送地址 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_MASTER 
} 

vrrp_script check_nginx {
    script "/usr/local/nginx/sbin/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state MASTER 
    interface em1
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 100    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        10.192.27.117/25 
    } 
    track_script {
        check_nginx
    } 
}

 

 

[root@nginx01 ~]# vi /usr/local/nginx/sbin/check_nginx.sh  #检测Nginx的脚本
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    systemctl stop keepalived
fi


chmod +x /usr/local/nginx/sbin/check_nginx.sh

 

启动服务

[root@nginx01 ~]# systemctl start keepalived  
[root@nginx01 ~]# systemctl enable keepalived
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
[root@nginx01 ~]# ps -ef | grep keepalived
root       7293      1  0 09:43 ?        00:00:00 /usr/sbin/keepalived -D
root       7294   7293  0 09:43 ?        00:00:00 /usr/sbin/keepalived -D
root       7295   7293  0 09:43 ?        00:00:00 /usr/sbin/keepalived -D
root       7462   7144  0 09:43 pts/0    00:00:00 grep --color=auto keepalived
[root@nginx01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:26:7c:a6 brd ff:ff:ff:ff:ff:ff
    inet 10.192.27.111/24 brd 192.168.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 10.192.27.117/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::6cf3:5b62:8b8:30d0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@nginx01 ~]# 

 

 

 

 

 

2. Nginx02配置

[root@nginx02 ~]# yum -y install keepalived
[root@nginx02 ~]# vi /etc/keepalived/keepalived.conf   keepalived配置文件
! configuration File for keepalived 
 
global_defs { 
   # 接收邮件地址 
   notification_email { 
     acassen@firewall.loc 
     failover@firewall.loc 
     sysadmin@firewall.loc 
   } 
   # 邮件发送地址 
   notification_email_from Alexandre.Cassen@firewall.loc  
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30 
   router_id NGINX_BACKUP
} 

vrrp_script check_nginx {
    script "/usr/local/nginx/sbin/check_nginx.sh"
}

vrrp_instance VI_1 { 
    state BACKUP 
    interface em1
    virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 
    priority 90    # 优先级，备服务器设置 90 
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒 
    authentication { 
        auth_type PASS      
        auth_pass 1111 
    }  
    virtual_ipaddress { 
        10.192.27.117/25 
    } 
    track_script {
        check_nginx
    } 
}

 

 

 

[root@nginx02 ~]# vi /usr/local/nginx/sbin/check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv "grep|$$")

if [ "$count" -eq 0 ];then
    systemctl stop keepalived
fi

chmod +x /usr/local/nginx/sbin/check_nginx.sh

 

 

启动服务

[root@nginx02 ~]# systemctl start keepalived
[root@nginx02 ~]# systemctl enable keepalived
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
[root@nginx02 ~]#  ps -ef | grep keepalived
root       7176      1  0 09:44 ?        00:00:00 /usr/sbin/keepalived -D
root       7177   7176  0 09:44 ?        00:00:00 /usr/sbin/keepalived -D
root       7178   7176  0 09:44 ?        00:00:00 /usr/sbin/keepalived -D
root       7272   7080  0 09:44 pts/0    00:00:00 grep --color=auto keepalived
[root@nginx02 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:d3:9b:90 brd ff:ff:ff:ff:ff:ff
    inet 10.192.27.112/24 brd 192.168.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::a582:7d1:9db7:ad5e/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@nginx02 ~]# 

 

 

 

3. 测试一下

[root@nginx01 ~]# pkill nginx  #主上
[root@nginx01 ~]# 
[root@nginx01 ~]# pkill nginx
[root@nginx01 ~]# ps -ef |grep nginx |egrep -cv "grep|$$"
0
###########
[root@nginx02 ~]# ip a   #备上
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:d3:9b:90 brd ff:ff:ff:ff:ff:ff
    inet 10.192.27.112/24 brd 192.168.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 10.192.27.117/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::a582:7d1:9db7:ad5e/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
############

#主上启动 刚才测试关掉的服务
[root@nginx01 ~]# systemctl restart nginx
[root@nginx01 ~]# systemctl restart keepalived
[root@nginx01 ~]# 
[root@nginx01 ~]# ip a      #VIP又回来
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:26:7c:a6 brd ff:ff:ff:ff:ff:ff
    inet 10.192.27.111/24 brd 192.168.1.255 scope global noprefixroute ens33
       valid_lft forever preferred_lft forever
    inet 10.192.27.117/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::6cf3:5b62:8b8:30d0/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@nginx01 ~]# 

 

 

 

四、修改node节点 10.192.27.115/10.192.27.116配置文件 

原来单master集群 时 node01和node02 默认连接10.192.27.100:6443（master01）

现在所有node节点指向 VIP：10.192.27.117，让其转发到 连接要NGINX负载集群中（10.192.27.100和10.192.27.114）

node01 

[root@node01 ~]# cd /opt/kubernetes/cfg/  # 以下三个配置文件server 10.192.27.100 都改成  10.192.27.117
[root@node01 cfg]#  ls *kubeconfig
bootstrap.kubeconfig  kubelet.kubeconfig  kube-proxy.kubeconfig
[root@node01 cfg]# vim bootstrap.kubeconfig 
[root@node01 cfg]# vim kubelet.kubeconfig 
[root@node01 cfg]# vim kube-proxy.kubeconfig 
[root@node01 cfg]# systemctl restart kubelet
[root@node01 cfg]# systemctl restart kube-proxy
[root@node01 cfg]# grep 100 *
bootstrap.kubeconfig:    server: https://10.192.27.117:6443
kubelet.kubeconfig:    server: https://10.192.27.117:6443
kube-proxy.kubeconfig:    server: https://10.192.27.117:6443

 

 

node02

[root@node02 ~]# cd /opt/kubernetes/cfg/
[root@node02 cfg]# ls *kubeconfig | xargs -i sed -i 's/10.192.27.100:6443/10.192.27.117:6443/' {}
[root@node02 cfg]# systemctl restart kubelet
[root@node02 cfg]# systemctl restart kube-proxy
[root@node02 cfg]# 

 

 

在主Nginx测试一下（因为node节点要连接kube-apiserver,经NGINX轮询转发）

[root@nginx01 ~]# tailf /var/log/nginx/k8s-access.log
10.192.27.115 10.192.27.114:6443, 10.192.27.100:6443 - [31/Aug/2019:10:02:35 +0800] 200 0, 1117
10.192.27.115 10.192.27.100:6443 - [31/Aug/2019:10:02:35 +0800] 200 1118
10.192.27.116 10.192.27.100:6443 - [31/Aug/2019:10:10:15 +0800] 200 1118
10.192.27.116 10.192.27.100:6443 - [31/Aug/2019:10:10:15 +0800] 200 1118