centos7搭建keepalived高可用nginx负载均衡集群

　　昨天在公司内部分享了高可用负载均衡，现将环境搭建过程记录一下。

　　本文主要讲keepalived高可用，与keepalived自检的脚本。

环境：
web01： 10.8.8.51   centos7.3　　#web服务器01
web02： 10.8.8.52   centos7.3　　#web服务器02
proxy01：10.8.8.90   centos7.3    vip：10.8.8.55　　#nginx负载均衡主服务器，keepalived主服务器
proxy02：10.8.8.91   centos7.3   vip：10.8.8.55　　#nginx负载均衡备服务器，keepalived备服务器

一、web01、web02环境搭建

　　由于本文主讲keepalived高可用，环境搭建过程忽略。

　　web01、web02各自搭建一个简单的web，域名均为：www.test.com

       分别测试单机访问是否成功。

二、配置nginx负载均衡

　　在10.8.8.90和10.8.8.91主机上，分别配置nginx负载均衡。简单配置如下。分别测试负载均衡是否成功。

upstream backend {
    server 10.8.8.51:80            max_fails=3 fail_timeout=30s;
    server 10.8.8.52:80            max_fails=3 fail_timeout=30s;
}

server {
    listen     80;
    server_name  www.test.com;
    location / {
    proxy_pass http://backend;
    proxy_set_header Host   $host;
    proxy_set_header X-Forwarded-For $remote_addr;
 }
}

三、安装配置keepalived

　　keepalived官网：www.keepalived.org

　　因为是测试用，所以用最新版本：

• Keepalived for Linux - Version 2.0.6 - July 23, 2018

　　下载地址：http://www.keepalived.org/software/keepalived-2.0.6.tar.gz

　　#iptables防火墙互相允许对方ip通过

 

#在proxy02机子上：
-I INPUT -s 10.8.8.90 -j ACCEPT
#在proxy01机子上：
-I INPUT -s 10.8.8.91 -j ACCEPT

 

　　1、安装keepalived

[root@proxy01 ~]# mkdir /home/tools
[root@proxy01 ~]# cd /home/tools/
[root@proxy01 tools]# wget http://www.keepalived.org/software/keepalived-2.0.6.tar.gz
[root@proxy01 tools]# tar xf keepalived-2.0.6.tar.gz 
[root@proxy01 tools]# cd keepalived-2.0.6
[root@proxy01 keepalived-2.0.6]# ./configure --prefix=/usr/local/keepalived
[root@proxy01 keepalived-2.0.6]# make
[root@proxy01 keepalived-2.0.6]# make install

　　#我的安装过lnmp环境，没有报错，如果编辑或安装时报错，先安装依赖。

yum install openssl-devel psmisc libnl* libnfnetlink-devel -y       #如果报错，就安装依赖。

　　2、配置keepalived为系统服务

　　#配置后即可使用/etc/init.d/keepalived启停keepalived服务，必须配置，在脚本中需要用到，如果使用service命令启停服务，crontab定时任务不会执行。

[root@proxy01 ~]# cp /usr/local/keepalived/sbin/keepalived /usr/sbin/
[root@proxy01 ~]# cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/
[root@proxy01 ~]# touch /etc/init.d/keepalived
[root@proxy01 ~]# chmod +x /etc/init.d/keepalived 
[root@proxy01 ~]# vi /etc/init.d/keepalived 

#/etc/init.d/keepalived内容如下：

#!/bin/sh
#
# Startup script for the Keepalived daemon
#
# processname: keepalived
# pidfile: /var/run/keepalived.pid
# config: /etc/keepalived/keepalived.conf
# chkconfig: - 21 79
# description: Start and stop Keepalived

# Source function library
. /etc/rc.d/init.d/functions

# Source configuration file (we set KEEPALIVED_OPTIONS there)
. /etc/sysconfig/keepalived

RETVAL=0

prog="keepalived"

start() {
    echo -n $"Starting $prog: "
    daemon keepalived ${KEEPALIVED_OPTIONS}
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && touch /var/lock/subsys/$prog
}

stop() {
    echo -n $"Stopping $prog: "
    killproc keepalived
    RETVAL=$?
    echo
    [ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$prog
}

reload() {
    echo -n $"Reloading $prog: "
    killproc keepalived -1
    RETVAL=$?
    echo
}

# See how we were called.
case "$1" in
    start)
        start
        ;;
    stop)
        stop
        ;;
    reload)
        reload
        ;;
    restart)
        stop
        start
        ;;
    condrestart)
        if [ -f /var/lock/subsys/$prog ]; then
            stop
            start
        fi
        ;;
    status)
        status keepalived
        RETVAL=$?
        ;;
    *)
        echo "Usage: $0 {start|stop|reload|restart|condrestart|status}"
        RETVAL=1
esac

exit $RETVAL

　　3、编辑keepalived配置文件

[root@proxy01 ~]# mkdir /etc/keepalived/   
[root@proxy01 ~]# cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
[root@proxy01 ~]# vi /etc/keepalived/keepalived.conf

　　#keepalived.conf主配置文件内容如下：

global_defs {
    router_id node_01   #id可以不同
}

vrrp_instance VI_1 {
    interface ens33
    state MASTER  # BACKUP for slave routers
    priority 101  # 100 for BACKUP
    virtual_router_id 51
    garp_master_delay 1
    authentication {
        auth_type PASS
        auth_pass password
    }
    track_interface {
       ens33
    }
    virtual_ipaddress {
        10.8.8.55/24
    }
}

　　4、启动keepalived

#脚本如果需要用启停命令，即使用/etc/init.d/keepalived start等方式。
$ service keepalived start          #启动keepalived
$ service keepalived stop           #停用keepalived
$ service keepalived status         #查看keepalived
$ service keepalived restart        #重启keepalived

[root@proxy01 ~]# ps -ef|grep keepalived | grep -v grep
[root@proxy01 ~]# /etc/init.d/keepalived start
Starting keepalived (via systemctl): [ OK ]
[root@proxy01 ~]# ps -ef|grep keepalived | grep -v grep
root 7558 1 0 16:13 ? 00:00:00 /usr/local/keepalived/sbin/keepalived -D
root 7559 7558 0 16:13 ? 00:00:00 /usr/local/keepalived/sbin/keepalived -D

#查看keepalived进程，有3个或2个-D说明成功

　　5、haproxy_backup机子安装keepalived

　　以同样的方法在proxy02服务器上安装nginx跟keepalived。
　　keepalived的backup配置文件相对于master修改如下：
　　state BACKUP
　　priority 50 

　　6、查看虚拟ip在哪台机子

　　#使用ip add查看虚拟ip是否启用

[root@proxy01 ~]# ip add sh ens33
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:e7:6a:1a brd ff:ff:ff:ff:ff:ff
    inet 10.8.8.90/24 brd 10.8.8.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 10.8.8.55/24 scope global secondary ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::32e6:c2a4:3a3f:904a/64 scope link 
       valid_lft forever preferred_lft forever
    inet6 fe80::b86b:7472:ada:f610/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::16e3:b3e1:8fd3:81e4/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever

　　四、keepalived脚本

　　将4台机子，全部配置完成。测试即可。

　　任意停用一台服务器，业务正常运转。但是。

　　假如在keepalived主服务器上停掉nginx，但是不停掉keepalived服务。此时，nginx服务不正常，但keepalived是正常的。

所以，会出现这样情况，导致业务无法正常访问。

　　所以要在proxy01上，写个脚本，内容为监控自身nginx和keepalived服务，如果nginx或者keepalived不正常，

即要自动做出相应的调整，让业务能够自动正常运转。

　　我写了个python脚本，内容如下，经过测试是可以在定时任务上自动执行的。

　　#proxy01监控自身nginx和keepalived脚本：

# keepalived的python脚本

#!/usr/bin/env python
# -*- conding:utf-8 -*-
import os
v = os.system("curl 127.0.0.1")
if v != 0: 
    nginx = os.system("/usr/local/tengine/sbin/nginx") 
    if nginx != 0:
        os.system("/etc/init.d/keepalived stop")
    else:
        k = os.system("netstat -anp|grep keepalived | grep 112")    
        if k != 0: 
            k2 = os.system("/etc/init.d/keepalived start") 
            if k2 != 0: 
                os.system("/etc/init.d/keepalived stop")  
else: 
    s = os.system("netstat -lntp|grep 80") 
    if s == 0:  
        k3 = os.system("netstat -anp|grep keepalived | grep 112")    
        if k3 != 0:  
            k4 = os.system("/etc/init.d/keepalived start")  
            if k4 != 0: 
                os.system("/etc/init.d/keepalived stop")

#!/usr/bin/env python
# -*- conding:utf-8 -*-
import os
v = os.system("curl 127.0.0.1")    # 测试本机nginx服务是否正常
if v != 0:  # 如果nginx不正常
    nginx = os.system("/usr/local/tengine/sbin/nginx")  # 启动nginx
    if nginx != 0:  # 如果启动nginx失败
        os.system("/etc/init.d/keepalived stop")    # 停用keepalived服务，ip自动跳转。
    else:   # 如果启动nginx成功
        k = os.system("netstat -anp|grep keepalived | grep 112")    # 检查keepalived是否正常
        if k != 0:  # 如果keepalived不正常
            k2 = os.system("/etc/init.d/keepalived start")  # 启动keepalived
            if k2 != 0:     # 如果启动keepalived失败
                os.system("/etc/init.d/keepalived stop")    # 停用keepalived，ip自动跳转
else:   # 如果nginx正常
    s = os.system("netstat -lntp|grep 80")  # 查看80端口
    if s == 0:  # 如果80端口正常
        k3 = os.system("netstat -anp|grep keepalived | grep 112")    # 检查keepalived服务是否正常
        if k3 != 0:  # 如果keepalived服务不正常
            k4 = os.system("/etc/init.d/keepalived start")   # 启动keepalived
            if k4 != 0:   # 如果启动keepalived失败
                os.system("/etc/init.d/keepalived stop") # 停用keepalived

经过测试定时任务是可以执行的。