10、Master高可用nginx+keepalived布署

Kubernetes学习目录

1、前言

# 这里因为演示，用master2、master3主机做为高可用布署

2、keepalived

2.1、安装

wget https://www.keepalived.org/software/keepalived-2.2.2.tar.gz --no-check-certificate
yum install gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl openssl-devel -y
tar xvf keepalived-2.2.2.tar.gz && cd keepalived-2.2.2 && ./configure --prefix=/usr/local/keepalived-2.2.2 && make && make install
cp /root/keepalived-2.2.2/keepalived/etc/init.d/keepalived /etc/init.d/
chmod 755 /etc/init.d/keepalived

2.2、安装后配置

# 1、修改脚本
[root@ ~]# vi /etc/init.d/keepalived 
...

# Set KEEPALIVED_OPTIONS
. /usr/local/keepalived-2.2.2/etc/sysconfig/keepalived

# 配置环境变量
export KEEPALIVED_HOME=/usr/local/keepalived-2.2.2
export PATH=${PATH}:${KEEPALIVED_HOME}/sbin
...

# 2、设置keepalived配置文件的位置
[root@ ~]# vi /usr/local/keepalived-2.2.2/etc/sysconfig/keepalived 
...
KEEPALIVED_OPTIONS="-D -f /usr/local/keepalived-2.2.2/etc/keepalived/keepalived.conf"

2.3、开机自启动

cat > /lib/systemd/system/keepalived.service <<'EOF'
[Unit]
Description=keepalived server daemon
Documentation=/usr/local/keepalived-2.2.2/
After=network.target

[Service]
Type=forking
ExecStart=/etc/init.d/keepalived start
ExecReload=/etc/init.d/keepalived reload
ExecStop=/etc/init.d/keepalived stop
Restart=/etc/init.d/keepalived restart
PrivateTmp=True

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable keepalived.service

2.4、Nginx健康检测试脚本

mkdir /usr/local/keepalived-2.2.2/script
cat >/usr/local/keepalived-2.2.2/script/check_nginx.sh <<'EOF'
#!/bin/bash
nginx_home=/usr/local/nginx-1.20.0/sbin/nginx

if [ $(ps -ef | grep -w $nginx_home | grep -v 'grep ' | wc -l) -eq 0 ];then
    #/etc/init.d/keepalived stop #Centos 6.X使用的方法
    /usr/bin/systemctl stop keepalived # Centos7.x使用的方法
echo "No Runing"
fi
EOF
chmod 755 /usr/local/keepalived-2.2.2/script/check_nginx.sh

3、Nginx

3.1、安装

wget http://nginx.org/download/nginx-1.20.1.tar.gz
yum install pcre  pcre-devel openssl  openssl-devel -y

useradd -s /sbin/nologin -M nginx
tar xvf nginx-1.20.1.tar.gz && cd nginx-1.20.1 && ./configure \
--user=nginx \
--group=nginx \
--prefix=/usr/local/nginx-1.20.0 \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-stream && make && make install

3.2、开机自启动

cat > /lib/systemd/system/nginx.service << 'EOF'
[Unit]
Description=nginx - high performance web server
After=network.target remote-fs.target nss-lookup.target

[Service]
Type=forking
PIDFile=/usr/local/nginx-1.20.0/logs/nginx.pid
ExecStartPre=/usr/local/nginx-1.20.0/sbin/nginx -t -c /usr/local/nginx-1.20.0/conf/nginx.conf
ExecStart=/usr/local/nginx-1.20.0/sbin/nginx -c /usr/local/nginx-1.20.0/conf/nginx.conf
ExecReload=/usr/local/nginx-1.20.0/sbin/nginx -s reload
ExecStop=/usr/local/nginx-1.20.0/sbin/nginx -s stop
ExecQuit=/usr/local/nginx-1.20.0/sbin/nginx -s quit
PrivateTmp=true

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable nginx

3.3、nginx配置文件优化

mkdir /usr/local/nginx-1.20.0/conf/conf.d


vi /usr/local/nginx-1.20.0/conf/nginx.conf
...
}
include  /usr/local/nginx-1.20.0/conf/conf.d/*.conf;
# 最末尾行

4、主Keepalived的配置

4.1、配置准备文件

ln -s /usr/local/keepalived-2.2.2 /usr/local/keepalived
cp /usr/local/keepalived-2.2.2/etc/keepalived/keepalived.conf /usr/local/keepalived-2.2.2/etc/keepalived/keepalived.conf.default

4.2、keepalived配置

! Configuration File for keepalived
! extra script call demonstration
! scripts are supported in Instance and groups
! declarations.

! router_id 是不一样的
global_defs {
   router_id K_2
   script_user root  
}

! 定义监控脚本位置
vrrp_script chk_nginx {
     script "/usr/local/keepalived-2.2.2/script/check_nginx.sh"
       interval 1
       weight 2
       fall 3
       rise 1
}

! 注意 virtual_router_id必须一致,priority数值越高，越优先
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    smtp_alert
! 非抢占
nopreempt
    virtual_router_id 1
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass iVZWO
    }

! 配置虚拟IP地址
    virtual_ipaddress {
        192.168.10.200 dev ens33 label ens33:1
    }
    ! 这里是上面检测脚本的名称
    track_script{
       chk_nginx
    }
}
主keepalived.conf

5、备Keepalived的配置

5.1、配置准备文件

ln -s /usr/local/keepalived-2.2.2 /usr/local/keepalived
cp /usr/local/keepalived-2.2.2/etc/keepalived/keepalived.conf /usr/local/keepalived-2.2.2/etc/keepalived/keepalived.conf.default

5.2、keepalived配置

! Configuration File for keepalived
! extra script call demonstration
! scripts are supported in Instance and groups
! declarations.

! router_id 是不一样的
global_defs {
   router_id K_1
   script_user root  
}

! 定义监控脚本位置
vrrp_script chk_nginx {
     script "/usr/local/keepalived/script/check_nginx.sh"
       interval 1
       weight 2
       fall 3
       rise 1
}

! 注意 virtual_router_id必须一致,priority数值越高，越优先
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    ! 非抢占，有nopreempt配置不能state是MASTER,必须都是BACKUP
nopreempt
    smtp_alert
    virtual_router_id 1
    priority 150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass iVZWO
    }

! 配置虚拟IP地址
    virtual_ipaddress {
        192.168.10.200 dev ens33 label ens33:1
    }

! 这里是上面检测脚本的名称
    track_script{
       chk_nginx
    }
}
备keepalived.conf

6、配置Nginx反向代理

6.1、配置背景

请求至VIP地址转发给master1、master2、master3进行处理

6.2、配置Nginx

6.2.1、http

# http反向代理的配置
cat >/usr/local/nginx-1.20.0/conf/conf.d/apiserver.conf<<'EOF'
upstream k8s-apiserver {
    server 192.168.10.26:6443 weight=2;
    server 192.168.10.27:6443 weight=1;
    server 192.168.10.28:6443 weight=1;
}
server {
    listen 6443;
    location / {
        proxy_pass http://k8s-apiserver;
        proxy_set_header           Host $host; 
        proxy_set_header           X-Real-IP $remote_addr; 
        proxy_set_header           X-Forwarded-For $proxy_add_x_forwarded_for; 
        client_max_body_size       10m; #允许客户端请求的最大单文件字节数
        client_body_buffer_size    128k; #缓冲区代理缓冲用户端请求的最大字节数
        proxy_connect_timeout      300; #nginx跟后端服务器连接超时时间(代理连接超时)
        proxy_send_timeout         300; #后端服务器数据回传时间(代理发送超时)
        proxy_read_timeout         300; #连接成功后，后端服务器响应时间(代理接收超时)
        proxy_buffer_size          4k; #设置代理服务器（nginx）保存用户头信息的缓冲区大小
        proxy_buffers              4 32k; #proxy_buffers缓冲区，网页平均在32k以下的话，这样设置
        proxy_busy_buffers_size    64k; #高负荷下缓冲大小（proxy_buffers*2）
        proxy_temp_file_write_size 64k; #设定缓存文件夹大小，大于这个值，将从upstream服务器传
    }
}
EOF

6.2.2、https【优先使用这个】

# https反向代理的配置【一般都是https优先使用这个】
cat >/usr/local/nginx-1.20.0/conf/conf.d/apiserver.conf<<'EOF' 
stream {
    upstream kube-apiserver {
        server 192.168.10.26:6443     max_fails=3 fail_timeout=30s;
        server 192.168.10.27:6443     max_fails=3 fail_timeout=30s;
        server 192.168.10.28:6443     max_fails=3 fail_timeout=30s;
    }
    server {
        listen 6443;
        proxy_connect_timeout 2s;
        proxy_timeout 900s;
        proxy_pass kube-apiserver;
    }
}
EOF

7、验证VIP自动漂移

7.1、启动nginx

systemctl start nginx

7.2、启动keepalived

# 两台机器都启动服务
systemctl restart keepalived

7.3、检查方法

# 关闭nginx
systemctl stop nginx

# 查询vip是否切换
posted @ 2023-03-16 11:28  小粉优化大师  阅读(51)  评论(0)  编辑  收藏  举报