Nginx HA
整体方案架构为:
(内网192.168.199.5)
+-----------VIP----------+
| |
| |
Master Backup
192.168.199.90 192.168.199.57
+----------+ +----------+
| HAProxy | | HAProxy |
|nginx(SSL)| |nginx(SSL)|
|keepalived| |keepalived|
+----------+ +----------+
|
v
192.168.199.88/89
+----------+
| multiple |
| NGINXs |
+----------+
|
v
+--------+---------+
| | |
| | |
v v v
+------+ +------+ +------+
| WEB1 | | WEB2 | | WEB3 |
+------+ +------+ +------+
各软件作用:
* Keepalived：判定HAProxy存活，保证HA
* HAProxy：做HTTP Load Balance
* Nginx(SSL)：与HAProxy放置在同一服务器，负责ssl offload
* Nginx(LB)：load balancer for app servers & web servers
客户端访问示意图:
+--------+ HTTP :80 +----------+
| client | --------------------------------> | |
| | | haproxy, |
+--------+ +---------+ | 1 or 2 |
/ / HTTPS | Nginx | HTTP :80 | listening|
<________/ ---------> | (SSL) | ---------> | ports |
| | | |
+---------+ +----------+
HAProxy + NGINX(SSL)
使用HAProxy做HTTP的Load Balancer，使用Nginx做SSL Offload。
测试环境:
* CentOS 6.4 x86_64 (Final)
* Supermicro 2U4 Node
* 域名: l99.com
IP分配:
* lb01.l99.com 192.168.199.88
* lb01.l99.com 192.168.199.89
* www.l99.com 192.168.199.5 (virtual IP)
* 192.168.199.90 做 Load Balancer (HAProxy + Nginx)
安装配置HAProxy
yum install libev-devel openssl-devel
cd /usr/local/src
wget http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.24.tar.gz
git clone https://github.com/cbonte/haproxy-patches.git
tar zxvf haproxy-1.4.24.tar.gz
# 给haproxy 1.4.24 打 proxy协议补丁(haproxy 1.5之后才支持accpet-proxy, 由于我们要使用stud做ssl offload, 需要支持accept-proxy)
cd haproxy-1.4.24
patch -p1 < /usr/local/src/haproxy-patches/proxy-protocol/haproxy-1.4-proxy-protocol.patch
make TARGETlinux2628 USE_EPOLL1 ARCHx86_64 && make install
cp /usr/local/src/haproxy-1.4.24/haproxy /usr/sbin/
cp examples/haproxy.init /etc/init.d/haproxy
chmod +x /etc/init.d/haproxy
chkconfig --add haproxy
chkconfig haproxy on
vim /etc/haproxy/haproxy.cfg
haproxy.cfg如下：
#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
# log 127.0.0.1 local2
log 127.0.0.1 local0
log 127.0.0.1 local1 debug
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 45000 # Total Max Connections. This is dependent on ulimit
user haproxy
group haproxy
daemon
nbproc 12 # 取决于CPU处理器核数，这里的测试机是2个6核Intel E5-2620 CPU，所以核数是12
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
balance roundrobin
# balance leastconn
option httplog
option dontlognull
option http-server-close
option forwardfor header X-Real-IP
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
timeout http-keep-alive 10s
timeout check 10s
maxconn 45000 # Total Max Connections. This is dependent on ulimit
stats enable
stats uri /stats # Real path redacted
stats realm Haproxy\ Statistics
stats auth username:password # Real credentials redacted
monitor-uri /monitor # Returns 200 if we're up; real path redacted
frontend http-in :80
reqdel X-Real-IP
reqadd X-Forwarded-Proto:\ http
default_backend http-load-balancer
frontend https-in
# bind 127.0.0.1:8443 accept-proxy
bind 127.0.0.1:8443
# reqdel X-Real-IP
reqadd X-Forwarded-Proto:\ https
default_backend http-load-balancer
backend http-load-balancer
server lb-1 192.168.199.88:80 maxconn 10000 check port 80
server lb-2 192.168.199.89:80 maxconn 10000 check port 80
安装配置Nginx(SSL)
/usr/local/nginx/conf/nginx.conf
user nginx;
worker_processes 12;
error_log logs/error.log crit;
pid logs/nginx.pid;
worker_rlimit_nofile 30000;
events {
use epoll;
worker_connections 51200;
}
http {
include mime.types;
default_type application/octet-stream;
# include common options #
include options.conf;
# include proxy settings #
include proxy.conf;
# domain config #
include l99.com/*.conf;
}
/usr/local/nginx/conf/l99.com/www.l99.com.conf
server {
listen 443;
ssl on;
ssl_certificate /usr/local/nginx/conf/l99.com/lifeix-l99.crt;
ssl_certificate_key /usr/local/nginx/conf/l99.com/lifeix-l99.key;
ssl_client_certificate /usr/local/nginx/conf/l99.com/lifeix-dvroot.crt;
ssl_session_timeout 5m;
ssl_protocols SSLv2 SSLv3 TLSv1;
ssl_ciphers ALL:!ADH:!EXPORT56:RC4+RSA:+HIGH:+MEDIUM:+LOW:+SSLv2:+EXP;
ssl_prefer_server_ciphers on;
default_type text/plain;
access_log logs/access.www.ssl.l99.com.log main;
error_log logs/error.www.ssl.l99.com.log;
server_name www.l99.com;
if ($request_uri ~ update.php) {
rewrite /(.*)$ http://www.L99.com/timeline.action last;
}
location / {
proxy_cache off;
proxy_next_upstream http_502 http_504 error timeout invalid_header;
proxy_ignore_headers Expires Cache-Control;
proxy_store off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
more_clear_headers "Cache-Control";
add_header Cache-Control "no-cache,max-age0";
proxy_pass http://127.0.0.1:8443;
}
}
启动并测试
service haproxy restart
service nginx restart
# 测试 HTTPS
openssl s_client -connect 192.168.199.90:443 -servername l99.com
# 测试HTTP
telnet 192.168.199.90 80
GET / HTTP/1.1
Host: www.L99.com
Nginx(LB)配置修改
修改options.conf (主要是由于使用HAProxy作为代理后，需要记录来源IP)
log_format main '$http_x_forwarded_proto $http_x_real_ip $remote_addr $host $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" "$http_user_agent" '
'$request_time $upstream_response_time $pipe "$gzip_ratio"';
重启nginx后，通过haproxy访问立方网日志如下:
https 192.168.199.15 192.168.199.90 www.l99.com - [04/Oct/2013:17:02:33 +0800] "GET /skin/recharge/images/paybtn_bg.jpg HTTP/1.1" 304 0 "https://www.l99.com/Recharge_pay.action" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.76 Safari/537.36" 0.007 0.006 . "-"
HAProxy + Keepalived
/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_haproxy {
script "killall -0 haproxy" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_script chk_nginx {
script "killall -0 nginx" # verify the pid existance
interval 2 # check every 2 seconds
weight 2 # add 2 points of prio if OK
}
vrrp_instance VI_1 {
interface eth0 # interface to monitor
state MASTER
virtual_router_id 51 # Assign one ID for this route
priority 101 # 101 on master, 100 on backup
virtual_ipaddress {
192.168.199.5 # the virtual IP
}
track_script {
chk_haproxy
chk_nginx
}
}
