keepalived实现nginx负载均衡机高可用
keepalived实现nginx负载均衡机高可用
环境说明
| 主机名称 | IP地址 | 系统版本 |
|---|---|---|
| master | 192.168.110.11 | redhat 8 |
| backup | 192.168.110.12 | redhat 8 |
本次高可用虚拟IP（VIP）地址暂定为 192.168.110.200
keepalived安装
配置主master
//master
#关闭防火墙和selinux
systemctl disable --now firewalld
sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/sysconfig/selinux
setenforce 0
#安装keepalived
[root@master ~]# yum -y install keepalived
配置备backup
//backup
#关闭防火墙和selinux
systemctl disable --now firewalld
sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/sysconfig/selinux
setenforce 0
#安装keepalived
[root@backup ~]# yum -y install keepalived
nginx安装
在主master上安装nginx
//master
#安装nginx
[root@master ~]# yum -y install nginx
#备份网页文件，创建新的测试网页
[root@master ~]# cd /usr/share/nginx/html/
[root@master html]# mv index.html{,.bak}
[root@master html]# echo 'master' > index.html
[root@master html]# ls
404.html 50x.html index.html index.html.bak nginx-logo.png poweredby.png
[root@master html]# cat index.html
master
#设置nginx开机自启
[root@master html]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@master html]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
#可以访问
[root@master html]# curl localhost
master
在备backup上安装nginx
//backup
#安装nginx
[root@backup ~]# yum -y install nginx
#备份网页文件，创建新的测试网页
[root@backup ~]# cd /usr/share/nginx/html/
[root@backup html]# mv index.html{,.bak}
[root@backup html]# echo 'backup' > index.html
[root@backup html]# ls
404.html 50x.html index.html index.html.bak nginx-logo.png poweredby.png
[root@backup html]# cat index.html
backup
#设置nginx开机自启
[root@backup html]# systemctl enable --now nginx
Created symlink /etc/systemd/system/multi-user.target.wants/nginx.service → /usr/lib/systemd/system/nginx.service.
[root@backup html]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
#可以访问
[root@backup html]# curl localhost
backup
keepalived配置
配置主keepalived
//master
#备份文件
[root@master ~]# cd /etc/keepalived/
[root@master keepalived]# mv keepalived.conf{,.bak}
[root@master keepalived]# ls
keepalived.conf keepalived.conf.bak
#配置文件
[root@master keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb01
}
vrrp_instance VI_1 {
state MASTER
interface ens160
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1
}
virtual_ipaddress {
192.168.110.200
}
}
virtual_server 192.168.110.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.110.11 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.110.12 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#设置开机自启
[root@master ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
配置备keepalived
//backup
#备份文件
[root@backup ~]# cd /etc/keepalived/
[root@backup keepalived]# mv keepalived.conf{,.bak}
[root@backup keepalived]# ls
keepalived.conf keepalived.conf.bak
#配置文件
[root@backup keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb02
}
vrrp_instance VI_1 {
state BACKUP
interface ens160
virtual_router_id 51
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1
}
virtual_ipaddress {
192.168.110.200
}
}
virtual_server 192.168.110.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.110.11 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.110.12 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#设置开机自启
[root@backup ~]# systemctl enable --now keepalived
Created symlink /etc/systemd/system/multi-user.target.wants/keepalived.service → /usr/lib/systemd/system/keepalived.service.
查看VIP在哪里
在master上查看
//master
#查看IP信息
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:ea:5b:7a brd ff:ff:ff:ff:ff:ff
inet 192.168.110.11/24 brd 192.168.110.255 scope global noprefixroute ens160
valid_lft forever preferred_lft forever
inet 192.168.110.200/32 scope global ens160 //此处可以看到VIP
valid_lft forever preferred_lft forever
在backup上查看
//backup
#查看IP信息
[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens160: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
link/ether 00:0c:29:85:c0:f0 brd ff:ff:ff:ff:ff:ff
inet 192.168.110.12/24 brd 192.168.110.255 scope global noprefixroute ens160
valid_lft forever preferred_lft forever
修改内核参数，开启监听VIP功能
此步可做可不做，该功能可用于仅监听VIP的时候
在master上修改内核参数
#master
[root@master ~]# echo 'net.ipv4.ip_nonlocal_bind = 1' >>/etc/sysctl.conf
[root@master ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@master ~]# cat /proc/sys/net/ipv4/ip_nonlocal_bind
1
在backup上修改内核参数
#slave
[root@backup ~]# echo 'net.ipv4.ip_nonlocal_bind = 1' >>/etc/sysctl.conf
[root@backup ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
[root@backup ~]# cat /proc/sys/net/ipv4/ip_nonlocal_bind
1
让keepalived监控nginx负载均衡机
keepalived通过脚本来监控nginx负载均衡机的状态
在master上编写脚本
此处的脚本名称应避免与服务名相同，推荐用服务名的首字母代替，如check_n，不要给脚本起名check_nginx
//master
#创建脚本目录
[root@master ~]# mkdir /scripts
[root@master ~]# cd /scripts/
#check_n 脚本
[root@master scripts]# vim check_n.sh
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
systemctl stop keepalived
fi
#notify 脚本
[root@master scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
subject="${VIP}'s server keepalived state is translate"
content="`date +'%F %T'`: `hostname`'s state change to master"
echo $content | mail -s "$subject" leidazhuang123@163.com
}
case "$1" in
master)
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
systemctl start nginx
fi
sendmail
;;
backup)
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -gt 0 ];then
systemctl stop nginx
fi
;;
*)
echo "Usage:$0 master|backup VIP"
;;
esac
#给脚本设置执行权限
[root@master scripts]# chmod +x check_n.sh
[root@master scripts]# chmod +x notify.sh
[root@master scripts]# ll
total 8
-rwxr-xr-x. 1 root root 147 May 24 11:05 check_n.sh
-rwxr-xr-x. 1 root root 663 May 24 11:06 notify.sh
在backup上编写脚本
//backup
#创建脚本目录
[root@backup ~]# mkdir /scripts
[root@backup ~]# cd /scripts/
#check_n 脚本
[root@backup scripts]# vim check_n.sh
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
systemctl stop keepalived
fi
# notify 脚本
[root@backup scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
subject="${VIP}'s server keepalived state is translate"
content="`date +'%F %T'`: `hostname`'s state change to master"
echo $content | mail -s "$subject" leidazhuang123@163.com
}
case "$1" in
master)
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
systemctl start nginx
fi
sendmail
;;
backup)
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -gt 0 ];then
systemctl stop nginx
fi
;;
*)
echo "Usage:$0 master|backup VIP"
;;
esac
#给脚本设置执行权限
[root@backup scripts]# chmod +x /scripts/check_n.sh
[root@backup scripts]# chmod +x notify.sh
[root@backup scripts]# ll
total 8
-rwxr-xr-x. 1 root root 147 May 24 11:08 check_n.sh
-rwxr-xr-x. 1 root root 663 May 24 11:08 notify.sh
配置keepalived，加入监控脚本的配置
配置主keepalived
//master
#添加配置
[root@master ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb01
}
vrrp_script nginx_check {
script "/scripts/check_n.sh"
interval 10
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface ens160
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1
}
virtual_ipaddress {
192.168.110.200
}
track_script {
nginx_check
}
notify_master "/scripts/notify.sh master 192.168.110.200"
notify_backup "/scripts/notify.sh backup 192.168.110.200"
}
virtual_server 192.168.110.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.110.11 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.110.12 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#重启服务
systemctl restart keepalived
配置备keepalived
//backup
#添加配置
[root@backup ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lb02
}
vrrp_instance VI_1 {
state BACKUP
interface ens160
virtual_router_id 51
priority 90
nopreempt
advert_int 1
authentication {
auth_type PASS
auth_pass 1
}
virtual_ipaddress {
192.168.110.200
}
notify_master "/scripts/notify.sh master 192.168.110.200"
notify_backup "/scripts/notify.sh backup 192.168.110.200"
}
virtual_server 192.168.110.200 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 50
protocol TCP
real_server 192.168.110.11 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.110.12 80 {
weight 1
TCP_CHECK {
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#重启服务
systemctl restart keepalived
模拟故障测试
启用keepalived，开启nginx服务
//master
#启用keepalived，开启nginx服务
[root@master ~]# systemctl start keepalived
[root@master ~]# systemctl start nginx
[root@master ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
//backup
#启用keepalived，不开启nginx服务，因为当master挂掉之后，会自动抢占
[root@backup ~]# systemctl start keepalived
[root@backup ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
模拟master主挂掉了，因为启用了高可用，backup备会自动继承
//模拟客户机
#nginx正常运行
[root@localhost ~]# curl 192.168.110.200
master
//master
#模拟master挂掉了，此时backup的nginx会自动启动
[root@master ~]# systemctl stop nginx
[root@master ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
//backup
#此时backup的nginx会自动启动
[root@backup ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
//此时用客户机访问VIP 192.168.110.200
#backup会自动接管nginx服务
[root@localhost ~]# curl 192.168.110.200
backup
此时我们模拟master被抢救回来，backup会停掉nginx；客户端访问VIP，此时会自动访问master
//master
#先启动nginx，再启动keepalived
[root@master ~]# systemctl start nginx
[root@master ~]# systemctl start keepalived
[root@master ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
[root@master ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 0.0.0.0:80 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
LISTEN 0 128 [::]:80 [::]:*
//客户端访问
#此时访问VIP会访问到master
[root@localhost ~]# curl 192.168.110.200
master
//backup
#backup会停掉nginx，自动让位
[root@backup ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 0.0.0.0:22 0.0.0.0:*
LISTEN 0 128 [::]:22 [::]:*
