LVS实现Kubernetes集群高可用

服务器规划：

三台k8s-master                                                     两台lvs                                      

k8s01：10.20.31.157                                            lb01：10.20.31.184

k8s02：10.20.31.167                                            lb02：10.20.31.185

k8s03：10.20.31.186                                            vipIP：10.20.31.187

从构图中可以看到，所有节点都需要通过负载均衡器和API Server进行通信，负载均衡器就非常重要了。这里考虑负载均衡器的性能与高可用，我们选择了LVS + keepalived（LVS当然也可以部署在k8s的节点机器上，为了保证集群高可用，建议还是部署在单独的机器上。）。

lvs-master(10.20.31.184)

# 安装依赖
$ yum install -y ipvsadm wget curl gcc openssl-devel libnl3-devel net-snmp-devel libnfnetlink-devel
# 安装keepalived，centos7通过yum下载的版本有问题，会报一个叫【TCP socket bind failed. Rescheduling】的错误
$ wget http://www.keepalived.org/software/keepalived-1.4.5.tar.gz && tar -zxvf keepalived-1.4.5.tar.gz && cd keepalived-1.4.5 && ./configure && make && make install && cd .. && rm -f keepalived-1.4.5.tar.gz && rm -rf keepalived-1.4.5
################ keepalived负载均衡配置 ################
# 生成keepalived配置
$ cd /etc/keepalived && cat <<E0F > /etc/keepalived/keepalived.conf
global_defs {
   router_id keepalived-master
}

vrrp_instance vip_1 {
  state MASTER
  ! 注意这是网卡名称，使用ip a命令查看自己的局域网网卡名称
  interface ens192
  ! keepalived主备router_id必须一致
  virtual_router_id 88
  ! 优先级，keepalived主节点优先级要比备节点高
  priority 100
  advert_int 3
  ! 配置虚拟ip地址
  virtual_ipaddress {
    10.20.31.187
  }
}

virtual_server 10.20.31.187 6443 {
  delay_loop 6
  lb_algo rr
  lb_kind DR
  persistence_timeout 0
  protocol TCP
    
  real_server 10.20.31.157 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.167 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.186 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
}
E0F

# 启动keepalived
$ systemctl enable keepalived && service keepalived start

# 检查keepalived状态
$ service keepalived status

# 查看日志
$ journalctl -f -u keepalived

# 查看虚拟ip
$ ip a

lvs-backup(10.20.31.185)

# 安装依赖
$ yum install -y ipvsadm wget curl gcc openssl-devel libnl3-devel net-snmp-devel libnfnetlink-devel
# 安装keepalived，centos7通过yum下载的版本有问题，会报一个叫【TCP socket bind failed. Rescheduling】的错误
$ wget http://www.keepalived.org/software/keepalived-1.4.5.tar.gz && tar -zxvf keepalived-1.4.5.tar.gz && cd keepalived-1.4.5 && ./configure && make && make install && cd .. && rm -f keepalived-1.4.5.tar.gz && rm -rf keepalived-1.4.5
################ keepalived负载均衡配置 ################
# 生成keepalived配置
$ mkdir -p /etc/keepalived && cd /etc/keepalived && cat <<E0F > /etc/keepalived/keepalived.conf
global_defs {
   router_id keepalived-backup
}

vrrp_instance vip_1 {
  state BACKUP
  ! 注意这是网卡名称，使用ip a命令查看自己的局域网网卡名称
  interface ens192
  ! keepalived主备router_id必须一致
  virtual_router_id 88
  ! 优先级，keepalived主节点优先级要比备节点高
  priority 99
  advert_int 3
  ! 配置虚拟ip地址
  virtual_ipaddress {
    10.20.31.187
  }
}

virtual_server 10.20.31.187 6443 {
  delay_loop 6
  lb_algo rr
  lb_kind DR
  persistence_timeout 0
  protocol TCP
    
  real_server 10.20.31.157 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.167 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
  real_server 10.20.31.186 6443 {
    weight 1
    TCP_CHECK {
      connect_timeout 10
      nb_get_retry 3
      delay_before_retry 3
      connect_port 6443
    }
  }
}
E0F

# 启动keepalived
$ systemctl enable keepalived && service keepalived start

# 检查keepalived状态
$ service keepalived status

# 查看日志
$ journalctl -f -u keepalived

# 查看虚拟ip
$ ip a

　

real_server配置，也就是每个k8s Master节点机器

# 创建rs脚本
$ mkdir -p /opt/rs/ && cd /opt/rs && cat <<E0F > /opt/rs/rs.sh
#!/bin/bash
# 虚拟ip
vip=10.20.31.187
# 停止以前的lo:0
ifconfig lo:0 down
echo "1" > /proc/sys/net/ipv4/ip_forward
echo "0" > /proc/sys/net/ipv4/conf/all/arp_announce
# 启动一个回环地址并绑定给vip
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
# ens33是主网卡名
echo "1" >/proc/sys/net/ipv4/conf/ens192/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/ens192/arp_announce
E0F

# 添加执行权限
$ chmod +x /opt/rs/rs.sh

# 执行rs脚本（如果出现错误，重新执行一遍即可）
$ ./rs.sh

# 添加到开机启动
$ echo '/opt/rs/rs.sh'  >> /etc/rc.d/rc.local
# 在centos7中，/etc/rc.d/rc.local的权限被降低了，所以需要执行如下命令赋予其可执行权限
$ chmod +x /etc/rc.d/rc.local