高可用集群搭建系列(二) keepalived与haproxy部署

一、环境准备

在进行keepalived与haproxy部署之前，需要先进行环境方面的准备：

节点名称	ip
master1	192.168.35.12
master2	192.168.35.13
node1	192.168.35.14
VIP	192.168.35.15

环境配置：

#1、 关闭防火墙
[root@localhost ~]# systemctl stop firewalld   # 临时关闭
[root@localhost ~]# systemctl disable firewalld # 永久关闭

#2、 关闭selinux
[root@localhost ~]# setenforce 0 # 临时关闭
[root@localhost ~]# sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久关闭

# 3、关闭swap
[root@localhost ~]# swapoff -a  #临时关闭
[root@localhost ~]# sed -ri 's/.*swap.*/#&/' /etc/fstab #永久关闭

#4、为每个节点设置主机名
[root@localhost ~]# hostnamectl set-hostname <hostname> # 例如master节点可以 　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　　
hostnamectl set-hostname master1

#5、在master节点上添加hosts
cat >> /etc/hosts << EOF
192.168.35.15 master.k8s.io k8s-vip
192.168.35.14 master01.k8s.io master1
192.168.35.13 master02.k8s.io master2
192.168.35.12 node01.k8s.io node1
EOF

#6、将桥接的IPv4流量传递到iptables的链
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system  # 生效

#7、 时间同步
yum install ntpdate -y
ntpdate time.windows.com

二、keepalived安装部署

1、安装相关依赖以及软件包

[root@master1 ~]# yum install -y conntrack-tools libseccomp libtool-ltdl
[root@master1 ~]# yum install -y keepalived

2、节点配置

 master1节点配置：

cat > /etc/keepalived/keepalived.conf <<EOF 
! Configuration File for keepalived

global_defs {
   router_id k8s
}

vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state MASTER 
    interface ens32 
    virtual_router_id 51
    priority 250
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.35.15
    }
    track_script {
        check_haproxy
    }

}
EOF

注意：

• vrrp_instance VI_1 中的interface是网卡，通过ifconfig进行具体查看各自网卡。
• virtual_ipaddress 虚拟ip这里设置的是 192.168.35.15

master2节点配置：

cat > /etc/keepalived/keepalived.conf <<EOF 
! Configuration File for keepalived

global_defs {
   router_id k8s
}

vrrp_script check_haproxy {
    script "killall -0 haproxy"
    interval 3
    weight -2
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP 
    interface ens32 
    virtual_router_id 51
    priority 200
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass ceb1b3ec013d66163d6ab
    }
    virtual_ipaddress {
        192.168.35.15
    }
    track_script {
        check_haproxy
    }

}
EOF

2、启动与检查

# 启动
[root@master1 ~]# systemctl start keepalived.service
# 检查
[root@master1 ~]# systemctl status keepalived.service

# 设置开机启动
[root@master1 ~]# systemctl enable keepalived.service

三、haproxy安装部署

1、安装haproxy软件

[root@master1 ~]# yum install -y haproxy

2、master节点配置

两个master节点后台配置相同，haproxy运行端口为16443，所以16443为集群的入口：

cat > /etc/haproxy/haproxy.cfg << EOF
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
    
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon 
       
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------  
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
#---------------------------------------------------------------------
# kubernetes apiserver frontend which proxys to the backends
#--------------------------------------------------------------------- 
frontend kubernetes-apiserver
    mode                 tcp
    bind                 *:16443
    option               tcplog
    default_backend      kubernetes-apiserver    
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server      master01.k8s.io   192.168.44.155:6443 check
    server      master02.k8s.io   192.168.44.156:6443 check
#---------------------------------------------------------------------
# collection haproxy statistics message
#---------------------------------------------------------------------
listen stats
    bind                 *:1080
    stats auth           admin:awesomePassword
    stats refresh        5s
    stats realm          HAProxy\ Statistics
    stats uri            /admin?stats
EOF

3、启动与查看

# 启动
[root@master1 ~]# systemctl start haproxy
# 查看
[root@master1 ~]# systemctl status haproxy
● haproxy.service - HAProxy Load Balancer
   Loaded: loaded (/usr/lib/systemd/system/haproxy.service; disabled; vendor preset: disabled)
   Active: active (running) since 二 2021-06-29 13:34:50 CST; 8s ago
...
# 查看端口
[root@master1 ~]# netstat -lntup|grep haproxy
tcp        0      0 0.0.0.0:1080            0.0.0.0:*               LISTEN      3357/haproxy        
tcp        0      0 0.0.0.0:16443           0.0.0.0:*               LISTEN      3357/haproxy        
udp        0      0 0.0.0.0:50010           0.0.0.0:*                           3356/haproxy        

设置开机启动：

[root@master1 ~]# systemctl enable haproxy
Created symlink from /etc/systemd/system/multi-user.target.wants/haproxy.service to /usr/lib/systemd/system/haproxy.service.