openresty用haproxy2.0实现负载均衡
安装openresty
编译安装
yum install pcre-devel openssl-devel gcc curl wget
wget https://openresty.org/download/openresty-1.15.8.3.tar.gz
tar -xzvf openresty-1.15.8.3.tar.gz
cd openresty-1.15.8.3
./configure
gmake
gmake install
yum 安装
yum install -y wget
wget -O /etc/yum.repos.d/openresty.repo https://openresty.org/package/centos/openresty.repo
yum install -y openresty openresty-resty
#默认安装编译参数检查
[root@localhost openresty-1.15.8.3]# openresty -V
nginx version: openresty/1.15.8.3
built by gcc 8.3.1 20190311 (Red Hat 8.3.1-3) (GCC)
built with OpenSSL 1.1.0l 10 Sep 2019
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt='-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include' --add-module=../ngx_devel_kit-0.3.1rc1 --add-module=../echo-nginx-module-0.61 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.31 --add-module=../ngx_lua-0.10.15 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.7 --with-ld-opt='-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib' --with-cc='ccache gcc -fdiagnostics-color=always' --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-stream_ssl_preread_module --with-http_ssl_module
#安装php
yum install php-fpm php-mysql php-xml php -y
systemctl enable php-fpm.service --now
#虚拟站点配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/test/index.php
echo hellowld
<h1>yayayayayale</h1>
[root@localhost conf]# cat /usr/local/openresty/nginx/conf.d/php.conf
server {
#监听端口
listen 801;
#网站根目录
root /usr/local/openresty/nginx/test;
#虚拟主机名称
server_name 192.168.168.21;
#网站主页排序
index index.php index.html index.htm default.php default.htm default.html;
#网站访问、错误日志
access_log /usr/local/openresty/nginx/test/test.access.log;
error_log /usr/local/openresty/nginx/logs/test/test.error.log;
#流量限制（网站最大并发数500|单IP访问最大并发数50|每个请求流量上限1024KB）
#limit_conn perserver 500;
#limit_conn perip 50;
#limit_rate 1024k;
#配置错误页面
#error_page 404 /404.html;
#error_page 500 502 503 504 /50x.html;
#禁止访问文件和目录
location ~ ^/(\.user.ini|\.htaccess|\.git|\.svn|\.project|LICENSE|README.md) {
return 404;
}
#配置资源防盗链
location ~ .*\.(jpg|jpeg|gif|png|js|css)$ {
expires 30d;
access_log /dev/null;
valid_referers none blocked 192.168.168.21;
if ($invalid_referer) {
return 404;
}
}
#配置图片资源缓存时间
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)$ {
expires 30d;
error_log off;
access_log /dev/null;
}
#设置样式资源缓存时间
location ~ .*\.(js|css)?$ {
expires 12h;
error_log off;
access_log /dev/null;
}
#解析PHP
location ~* \.php$ {
fastcgi_index index.php;
fastcgi_pass 127.0.0.1:9000;
include fastcgi_params;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
fastcgi_param SCRIPT_NAME $fastcgi_script_name;
}
}
#主配置文件
[root@localhost conf]# cat /usr/local/openresty/nginx/conf/nginx.conf
#user nobody;
worker_processes 1;
error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
events {
worker_connections 1024;
}
http {
#加载虚拟主机配置文件
include /usr/local/openresty/nginx/conf.d/*.conf;
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
log_format json '{"@timestamp":"$time_iso8601",'
'"@version":"1",'
'"client":"$remote_addr",'
'"url":"$uri",'
'"status":"$status",'
'"domain":"$host",'
'"host":"$server_addr",'
'"size":"$body_bytes_sent",'
'"responsentime":"$request_time",'
'"referer":"$http_referer",'
'"useragent":"$http_user_agent",'
'"upstreampstatus":"$upstream_status",'
'"upstreamaddr":"$upstream_addr",'
'"upstreamresponsetime":"$upstream_response_time"'
'}';
access_log logs/access_json.log json;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index1.html index.htm;
}
#error_page 404 /404.html;
# redirect server error pages to the static page /50x.html
#
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
# proxy the PHP scripts to Apache listening on 127.0.0.1:80
#
#location ~ \.php$ {
# proxy_pass http://127.0.0.1;
#}
# pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
#
#location ~ \.php$ {
# root html;
# fastcgi_pass 127.0.0.1:9000;
# fastcgi_index index.php;
# fastcgi_param SCRIPT_FILENAME /scripts$fastcgi_script_name;
# include fastcgi_params;
#}
# deny access to .htaccess files, if Apache's document root
# concurs with nginx's one
#
#location ~ /\.ht {
# deny all;
#}
}
# another virtual host using mix of IP-, name-, and port-based configuration
#
#server {
# listen 8000;
# listen somename:8080;
# server_name somename alias another.alias;
# location / {
# root html;
# index index.html index.htm;
# }
#}
# HTTPS server
#
#server {
# listen 443 ssl;
# server_name localhost;
# ssl_certificate cert.pem;
# ssl_certificate_key cert.key;
# ssl_session_cache shared:SSL:1m;
# ssl_session_timeout 5m;
# ssl_ciphers HIGH:!aNULL:!MD5;
# ssl_prefer_server_ciphers on;
# location / {
# root html;
# index index.html index.htm;
# }
#}
}
安装haproxy
1.1需要先安装lua5.3版本，否则后面后报错
curl -R -O http://www.lua.org/ftp/lua-5.3.5.tar.gz
yum install readline-devel gcc -y
tar zxvf lua-5.3.5.tar.gz
cd lua-5.3.5
make linux
make INSTALL_TOP=/usr/local/lua install
yum install systemd-devel wget openssl openssl-devel -y
1.2部署haproxy。
cdwget https://www.haproxy.org/download/2.0/src/haproxy-2.0.1.tar.gztar zxvf haproxy-2.0.1.tar.gz
cd haproxy-2.0.1
make -j $(nproc) TARGET=linux-glibc USE_OPENSSL=1 USE_ZLIB=1 USE_LUA=1 LUA_LIB=/usr/local/lua/lib/ LUA_INC=/usr/local/lua/include/ USE_PCRE=1 USE_SYSTEMD=1
make install PREFIX=/usr/local/haproxycd
1.3创建运行用户（应该也可忽略，只是使用该用户运行，加强安全而已。我一般是直接使用root用户运行）
useradd haproxy -s /sbin/nologin
1.4创建配置文件haproxy.cfg。在安装目录/usr/local/hadproxy下新建haproxy.cfg
global #全局设置
daemon #以后台进程运行
maxconn 256 #每个进程的最大连接数
nbproc 1 #进程数，该值可以设置小于或等于cpu核心数
balance roundrobin #默认的负载均衡的方式,轮询方式
#balance source #默认的负载均衡的方式,类似nginx的ip_hash
#balance leastconn #默认的负载均衡的方式,最小连接
defaults #默认设置
mode http #设置http(七层模式)，也可设置为tcp(四层模式)，另外还有一个Health健康监测模式。对mysql进行负载均衡的话，这里记得修改为tcp
timeout connect 5000ms
timeout client 50000ms
timeout server 50000ms
listen admin_stats #配置haproxy管理页面
bind *:9999 #访问端口为9999
mode http
option httplog
stats refresh 30s #自动刷新时间
stats uri /stats #项目名为status，ip+端口+项目名即可访问
stats auth admin:admin #配置管理用户账号密码
stats admin if TRUE
stats hide-version
frontend http-in #配置前端访问端口
bind *:1080 #通过该端口进行负载均衡
default_backend servers #指定后端服务器
backend servers
server server1 172.16.1.230:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32
#建议加上check，否则后台服务器A宕机了，负载均衡还会把请求发送到该宕机服务器上，inter 2000指check检测时间为2000毫秒，rise 3检测3次均正常则表示后天服务器正常，fall 3检测3次失败，则会把该后天服务器标志宕机，不再玩该后台服务器发送请求，weight 1指权重，取消weight改为backup，则是所有后台服务器宕机后才会启用该backup后台服务器
server server2 172.16.1.227:80 check inter 2000 rise 3 fall 3 weight 1 maxconn 32
#参考yum 安装的配置文件
[root@node2 haproxy]# cat /etc/haproxy/haproxy.cfg |grep -v "^#"
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
frontend main *:5000
acl url_static path_beg -i /static /images /javascript /stylesheets
acl url_static path_end -i .jpg .gif .png .css .js
use_backend static if url_static
default_backend app
backend static
balance roundrobin
server static 127.0.0.1:4331 check
backend app
balance roundrobin
server app1 127.0.0.1:5001 check
server app2 127.0.0.1:5002 check
server app3 127.0.0.1:5003 check
server app4 127.0.0.1:5004 check
1.5使用指定配置文件运行haproxy。建议都haproxy和hadproxy.cfg都使用绝对路径
#/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg
1.6验证安装的haproxy版本
[root@localhost ~]# /usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 2.0.1 2019/06/26 - https://haproxy.org/
1.7配置开机自启
#修改haproxy cfg
[root@localhost haproxy]# cat /usr/local/haproxy/haproxy.cfg
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /usr/local/haproxy
pidfile /usr/local/haproxy/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /usr/local/haproxy/socket
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
frontend main
bind 0.0.0.0:8000
acl url_static path_beg -i /static /images /javascript /stylesheets
acl url_static path_end -i .jpg .gif .png .css .js
use_backend static if url_static
default_backend app
backend static
balance roundrobin
server static 127.0.0.1:4331 check
backend app
balance roundrobin
server app1 127.0.0.1:80 check
server app2 127.0.0.1:801 check
#server app3 127.0.0.1:5003 check
#server app4 127.0.0.1:5004 check
[root@localhost haproxy]#
[root@localhost haproxy]# cat /usr/lib/systemd/system/haproxy.service
[Unit]
Description=haporxy load bulancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg -c -q
ExecStart=/usr/local/haproxy/sbin/haproxy -Ws -f /usr/local/haproxy/haproxy.cfg -p /usr/local/haproxy/haproxy.pid
ExecReload=/bin/kill -USER2 $MAINPID
[Install]
WantedBy=multi-user.target
systemctl enable haproxy --now
1.8测试效果
其中228的80端口上我启用了httpd，页面内容为this is 228.另外一台230上的80端口也是httpd，页面为默认内容。使用谷歌浏览器访问172.16.1.227，每次刷新的内容都和上一次不一样
curl 127.0.0.1:8000
[root@localhost haproxy]# curl 192.168.1.119:8000
<!DOCTYPE html>
<html>
<head>
<title>Welcome to OpenResty!</title>
<style>
body {
width: 35em;
margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif;
}
</style>
</head>
<body>
<h1>Welcome to OpenResty!</h1>
<p>If you see this page, the OpenResty web platform is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="https://openresty.org/">openresty.org</a>.<br/>
Commercial support is available at
<a href="https://openresty.com/">openresty.com</a>.</p>
<p><em>Thank you for flying OpenResty.</em></p>
</body>
</html>
[root@localhost haproxy]# curl 192.168.1.119:8000
echo hellowld
<h1>yayayayayale</h1>
如果当你发现自己的才华撑不起野心时，那就请你安静下来学习
