五、LVS+Keepalive+nginx实验

一、实验环境

[root@node1 ~]# uname -r
3.10.0-862.el7.x86_64
[root@node1 ~]# cat /etc/redhat-release 
CentOS Linux release 7.5.1804 (Core) 

lvs+keepalived master:inode1: 192.168.31.101
lvs+keepalived backup:inode2: 192.168.31.102
nginx01:inode3：192.168.31.103----页面信息 www.ywx3.com
nginx02:inode4：192.168.31.104----页面信息 www.ywx4.com

VIP:192.168.31.111

二、LVS+Keepalived+nginx的部署

1、nginx部署

nginx01(inode3)和nginx02(inode4)

yum install -y nginx
systemctl nginx start

#nginx01的页面信息
echo 'www.ywx3.com' > /usr/local/nginx/html/index.html

#nginx02的页面信息
echo 'www.ywx4.com' > /usr/local/nginx/html/index.html

[root@node3 ~]# curl 192.168.31.103
www.ywx3.com

[root@node4 ~]# curl 192.168.31.104
www.ywx4.com

2、部署lvs+keepalived

inode1和inode2

 yum install -y ipvsadm keepalived 

3、编写lvs+keepalived的配置文件

lvs+keepalived master

! Configuration File for keepalived
global_defs {
   notification_email {
      yaowangxi@163.com
   }
   notification_email_from yaowangxi@163.com
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30
   router_id LVS_Keepalived_Master
}
# VIP1
vrrp_instance VI_1 {
    state  MASTER  
    interface  ens160
    lvs_sync_daemon_inteface ens160
    virtual_router_id 51
    priority 100 
    advert_int 5
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111   
    }
}
virtual_server 192.168.31.111 80 {
    delay_loop 6    
    lb_algo rr    
    lb_kind DR   
    persistence_timeout  60   
    protocol TCP     
    real_server 192.168.31.103 80 {
        weight 100        
        TCP_CHECK {
        connect_timeout 10 
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {
        weight 100
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
}

lvs+keepalived backup

! Configuration File for keepalived
global_defs {
   notification_email {
      yaowangxi@163.com
   }
   notification_email_from yaowangxi@163.com
   smtp_server 127.0.0.1 
   smtp_connect_timeout 30
   router_id LVS_Keepalived_Backup
}
# VIP1
vrrp_instance VI_1 {
    state  BACKUP 
    interface  ens160
    lvs_sync_daemon_inteface ens160
    virtual_router_id 51
    priority 90 
    advert_int 5
    nopreempt
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111   
    }
}
virtual_server 192.168.31.111 80 {
    delay_loop 6    
    lb_algo rr    
    lb_kind DR   
    persistence_timeout  60   
    protocol TCP     
    real_server 192.168.31.103 80 {
        weight 1        
        TCP_CHECK {
        connect_timeout 10 
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {
        weight 1
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
}

4、配置nginx01和nginx02上的VIP并抑制arp广播

配置VIP，nginx01(inode3)和nginx02(inode4)操作一样

cd /etc/sysconfig/network-scripts/
cp cp ifcfg-lo ifcfg-lo:0
vim ifcfg-lo:0

DEVICE=lo:0
IPADDR=192.168.31.111
NETMASK=255.255.255.255
ONBOOT=yes
NAME=loopback

ifup ifcfg-lo:0

nginx01(inode3)
[root@node3 network-scripts]# ifconfig|sed -n '/lo:0/,$p'
lo:0: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 192.168.31.111  netmask 255.255.255.255
        loop  txqueuelen 1000  (Local Loopback)

nginx02(inode4)
[root@node4 network-scripts]#  ifconfig|sed -n '/lo:0/,$p'
lo:0: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 192.168.31.111  netmask 255.255.255.255
        loop  txqueuelen 1000  (Local Loopback)

配置arp抑制nginx01(inode3)和nginx02(inode4)操作一样

echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p 

5、启动keepalived

lvs+keepalived master(inode1)和lvs+keepalived backup(inode2)

 systemctl start keepalived 

6、查看VIP地址

VIP地址在inode1上
[root@node1 ~]# ip addr list|grep -w "192.168.31.111"
    inet 192.168.31.111/32 scope global ens160

7、查看ipvsadm并访问vip

在inode1上查看
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   100    0          0         
  -> 192.168.31.104:80            Route   100    0          0       

[root@node3 network-scripts]# curl 192.168.31.111
www.ywx3.com

[root@node4 network-scripts]# curl 192.168.31.111
www.ywx4.com

8、关闭nginx01,查看ipvsadm（inode1上）

inode3
[root@node3 ~]# nginx -s stop
[root@node3 ~]# ps -ef |grep 'nginx'
root     17695 17147  0 08:05 pts/0    00:00:00 grep --color=auto nginx

inode1
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.104:80            Route   100    0          1  
#RS nginx01(inode3)192.168.31.103已经被移除

9、恢复nginx01,inode3会自动添加进ipvsadm

nginx01(inode3)
[root@node3 ~]# nginx
[root@node3 ~]# ps -ef |grep 'nginx'
root     17699     1  0 08:07 ?        00:00:00 nginx: master process nginx
nginx    17700 17699  0 08:07 ?        00:00:00 nginx: worker process
root     17702 17147  0 08:07 pts/0    00:00:00 grep --color=auto nginx


inode1
[root@node1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   1      0          0         
  -> 192.168.31.104:80            Route   100    0          0   

10、关闭inode1的keepalived,VIP会自动漂移到inode2上

inode1
[root@node1 ~]# ip addr list|grep -w '192.168.31.111'
    inet 192.168.31.111/32 scope global ens160
[root@node1 ~]# systemctl stop keepalived
[root@node1 ~]# ip addr list|grep -w '192.168.31.111'
[root@node1 ~]# 

inode2
[root@node2 ~]# ip addr list|grep -w '192.168.31.111'
    inet 192.168.31.111/32 scope global ens160
[root@node2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.31.111:80 rr persistent 60
  -> 192.168.31.103:80            Route   1      0          0         
  -> 192.168.31.104:80            Route   1      0          0         

三、lvs+keepalived的配置文件解析

#keepalived部分
global_defs {
   notification_email {
         yaowangxi@163.com
   }
   notification_email_from 441520481@qq.com
   smtp_server 192.168.80.1
   smtp_connection_timeout 30
   router_id LVS_DEVEL  # 设置lvs的id，在一个网络内应该是唯一的
}
vrrp_instance VI_1 {
    state MASTER #指定Keepalived的角色，MASTER为主，BACKUP为备 记得大写
    interface eno160  #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
    virtual_router_id 51  #虚拟路由编号，主备要一致
    priority 100  #定义优先级，数字越大，优先级越高，主DR必须大于备用DR
    advert_int 1  #检查间隔，默认为1s
    authentication {   #这里配置的密码最多为8位，主备要一致，否则无法正常通讯
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.31.111  #定义虚拟IP(VIP)为192.168.31.111，可多设，每行一个
    }
}
#LVS部分
# 定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.31.111 80 {
    delay_loop 6 # 设置健康检查时间，单位是秒
    lb_algo rr # 设置负载调度的算法为wlc
    lb_kind DR # 设置LVS实现负载的机制，有NAT、TUN、DR三个模式
    nat_mask 255.255.255.0
    persistence_timeout 0
    protocol TCP
    real_server 192.168.31.103 80 {  # 指定real server1的IP地址
        weight 1   # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.31.104 80 {  # 指定real server2的IP地址
        weight 1  # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
     }
}

四、realserver脚本文件

#虚拟的vip 根据自己的实际情况定义
SNS_VIP=192.168.31.111
/etc/rc.d/init.d/functions
case "$1" in
start)
       ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
       /sbin/route add -host $SNS_VIP dev lo:0
       echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
       sysctl -p >/dev/null 2>&1
       echo "RealServer Start OK"
       ;;
stop)
       ifconfig lo:0 down
       route del $SNS_VIP >/dev/null 2>&1
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
       echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
       echo "RealServer Stoped"
       ;;
*)
       echo "Usage: $0 {start|stop}"
       exit 1
esac
exit 0