Kubernetes nginx 负载配置
1.nginx负载连接master的双节点拓扑
2.配置官方yum源
cat > /etc/yum.repos.d/nginx.repo << EOF [nginx] name=nginx repo baseurl=http://nginx.org/packages/centos/7/$basearch/ gpgcheck=0 EOF
[root@dn05 ~]# yum clean all
[root@dn05 ~]# yum makecache
3.安装配置第一台nginx服务器
3.1 yum 安装nginx
[root@dn05 ~]# yum install -y nginx
3.2配置nginx文件
配置四层负载，加入负载的节点及监听的端口信息，这里负载的节点时kubernetes 的master01和master02 的地址
[root@dn05 ~]# cat /etc/nginx/nginx.conf user nginx; worker_processes 1; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { worker_connections 1024; } stream { log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; access_log /var/log/nginx/k8s-access.log main; upstream k8s-apiserver { server 10.10.100.30:6443; server 10.10.100.33:6443; } server { listen 6443; proxy_pass k8s-apiserver; } } http { include /etc/nginx/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; #tcp_nopush on; keepalive_timeout 65; #gzip on; include /etc/nginx/conf.d/*.conf; }
注意：在配置nginx 负载监听的端口时，要将selinux关闭，否在，在启动nginx时（即使没有被占用）会报错类似端口错误，导致无法启动nginx
将nginx_master的欢迎页修改为"nginx master",以便区分主节点。
[root@dn05 ~]# echo "nginx master "> /usr/share/nginx/html/index.html
启动nginx并设置开机启动
[root@dn05 ~]systemctl start nginx
[root@dn05 ~]systemctl enable nginx
测试nginx_master欢迎页
3.3相同的方法配置nginx_back节点
修改nginx_back的配置文件内容相同
[root@dn06 ~]# cat /etc/nginx/nginx.conf user nginx; worker_processes 1; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; events { worker_connections 1024; } stream { log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent'; access_log /var/log/nginx/k8s-access.log main; upstream k8s-apiserver { server 10.10.100.30:6443; server 10.10.100.33:6443; } server { listen 6443; proxy_pass k8s-apiserver; } } http { include /etc/nginx/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /var/log/nginx/access.log main; sendfile on; #tcp_nopush on; keepalive_timeout 65; #gzip on; include /etc/nginx/conf.d/*.conf; }
同样，将nginx_backup的欢迎页修改为“nginx backup”，以便区分备节点。
[root@dn06 ~]# echo "nginx backup "> /usr/share/nginx/html/index.html
测试nginx_back欢迎页
4.nginx 负载高可用keepalived配置
4.1 通过yum安装
首先在nginx_master配置keepalived
[root@dn05 ~]# yum install -y keepalived
4.2 备份keepalived的配置文件，为简化配置使用如下配置文件内容：
[root@dn05 keepalived]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { # 接收邮件地址 notification_email { acassen@firewall.loc failover@firewall.loc sysadmin@firewall.loc } # 邮件发送地址 notification_email_from Alexandre.Cassen@firewall.loc smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGINX_MASTER } vrrp_script check_nginx { script "/usr/local/nginx/sbin/check_nginx.sh" } vrrp_instance VI_1 { state MASTER interface ens33 virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 priority 100 # 优先级，备服务器设置 90 advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.10.100.37/24 } track_script { check_nginx } }
根据配置内容创建检查nginx的脚本
[root@dn05 ~]# mkdir /usr/local/nginx/sbin/
[root@dn05 ~]# vi /usr/local/nginx/sbin/check_nginx.sh
[root@dn05 ~]# cat /usr/local/nginx/sbin/check_nginx.sh count=$(ps -ef |grep nginx |egrep -cv "grep|$$") if [ "$count" -eq 0 ];then systemctl stop keepalived fi
[root@dn05 ~]# chmod +x /usr/local/nginx/sbin/check_nginx.sh
4.3启动keepalived
[root@dn05 ~]systemctl start keepalived
[root@dn05 ~]systemctl enable keepalived
同样的方式配置nginx_backup的keekpalived，注意配置文件的设置
[root@dn06 ~]# cat /etc/keepalived/keepalived.conf ! Configuration File for keepalived global_defs { # 接收邮件地址 notification_email { acassen@firewall.loc failover@firewall.loc sysadmin@firewall.loc } # 邮件发送地址 notification_email_from Alexandre.Cassen@firewall.loc smtp_server 127.0.0.1 smtp_connect_timeout 30 router_id NGINX_MASTER } vrrp_script check_nginx { script "/usr/local/nginx/sbin/check_nginx.sh" } vrrp_instance VI_1 { state BACKUP interface ens33 virtual_router_id 51 # VRRP 路由 ID实例，每个实例是唯一的 priority 90 # 优先级，备服务器设置 90 advert_int 1 # 指定VRRP 心跳包通告间隔时间，默认1秒 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 10.10.100.37/24 } track_script { check_nginx } }
将检查nginx的脚本从nginx_master上拷贝过来
[root@dn06 ~]# scp root@10.10.100.34:/usr/local/nginx/sbin/check_nginx.sh /usr/local/nginx/sbin/
[root@dn06 ~]# chmod +x /usr/local/nginx/sbin/check_nginx.sh
启动keepalived
[root@dn05 ~]systemctl start keepalived
[root@dn05 ~]systemctl enable keepalived
查看网卡信息
Nginx_Master IP: [root@dn05 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:8f:aa:8a brd ff:ff:ff:ff:ff:ff inet 10.10.100.34/24 brd 10.10.100.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet 10.10.100.37/24 scope global secondary ens33 valid_lft forever preferred_lft forever inet6 fe80::389d:e340:ea17:3a30/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::25e8:8754:cb81:68c8/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::1534:7f05:3d6a:9287/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever [root@dn05 ~]# Nginx_Backup IP: [root@dn06 sbin]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:d2:82:1e brd ff:ff:ff:ff:ff:ff inet 10.10.100.35/24 brd 10.10.100.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet6 fe80::389d:e340:ea17:3a30/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::25e8:8754:cb81:68c8/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::1534:7f05:3d6a:9287/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever [root@dn06 sbin]#
查看可以看到VIP在nginx_master节点上
测试
杀死主节点Nginx_Master的nginx进程，模拟主节点故障
[root@dn05 ~]# pkill nginx [root@dn05 ~]# ps -ef | grep nginx root 13040 7353 0 18:45 pts/0 00:00:00 grep --color=auto nginx [root@dn05 ~]# ps -ef | grep keep root 13042 7353 0 18:45 pts/0 00:00:00 grep --color=auto keep [root@dn05 ~]# 根据查看的结果nginx已经关闭，并且脚本已经执行，keepalived已被关闭
查看机器master 的IP，VIP已经转移
[root@dn05 ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:8f:aa:8a brd ff:ff:ff:ff:ff:ff inet 10.10.100.34/24 brd 10.10.100.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet6 fe80::389d:e340:ea17:3a30/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::25e8:8754:cb81:68c8/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::1534:7f05:3d6a:9287/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever
查看Nginx_Backup的IP，VIP已经漂移过来
[root@dn06 sbin]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 link/ether 00:0c:29:d2:82:1e brd ff:ff:ff:ff:ff:ff inet 10.10.100.35/24 brd 10.10.100.255 scope global noprefixroute ens33 valid_lft forever preferred_lft forever inet 10.10.100.37/24 scope global secondary ens33 valid_lft forever preferred_lft forever inet6 fe80::389d:e340:ea17:3a30/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::25e8:8754:cb81:68c8/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever inet6 fe80::1534:7f05:3d6a:9287/64 scope link tentative noprefixroute dadfailed valid_lft forever preferred_lft forever
访问vip，nginx首页已经跳转到备节点上
