Nginx 四层负载均衡

Nginx在1.9.0版本开始支持tcp模式的负载均衡，在1.9.13版本开始支持udp协议的负载，udp主要用于DNS的域名解析，其配置方式和指令和http 代理类似，其基于ngx_stream_proxy_module模块实现tcp负载，另外基于模块ngx_stream_upstream_module实现后端服务器分组转发、权重分配、状态监测、调度算法等高级功能。

如果编译安装,需要指定 --with-stream 选项才能支持ngx_stream_proxy_module模块

查看是否添加该模块:nginx -V,搜索是否包含--with-stream,如果包含则已添加过此模块。

使用yum方式安装的openresty带有该模块

wget https://openresty.org/package/centos/openresty.repo && mv openresty.repo /etc/yum.repos.d/ && yum makecache
yum -y install openresty
mkdir -p /var/log/nginx
systemctl start openresty.service && systemctl enable openresty.service

# cd /usr/local/openresty/nginx/sbin/
# ./nginx -V
nginx version: openresty/1.19.3.2
built by gcc 10.2.1 20200804 (Red Hat 10.2.1-2) (GCC) 
built with OpenSSL 1.1.1k  25 Mar 2021
TLS SNI support enabled
configure arguments: --prefix=/usr/local/openresty/nginx --with-cc-opt='-O2 -DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/zlib/include -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl111/include' --add-module=../ngx_devel_kit-0.3.1 --add-module=../echo-nginx-module-0.62 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.32 --add-module=../ngx_lua-0.10.19 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis2-nginx-module-0.15 --add-module=../redis-nginx-module-0.3.7 --add-module=../ngx_stream_lua-0.0.9 --with-ld-opt='-Wl,-rpath,/usr/local/openresty/luajit/lib -L/usr/local/openresty/zlib/lib -L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl111/lib -Wl,-rpath,/usr/local/openresty/zlib/lib:/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl111/lib' --with-cc='ccache gcc -fdiagnostics-color=always' --with-pcre-jit --with-stream --with-stream_ssl_module --with-stream_ssl_preread_module --with-http_v2_module --without-mail_pop3_module --without-mail_imap_module --without-mail_smtp_module --with-http_stub_status_module --with-http_realip_module --with-http_addition_module --with-http_auth_request_module --with-http_secure_link_module --with-http_random_index_module --with-http_gzip_static_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-threads --with-compat --with-stream --with-http_ssl_module

nginx相关网址

官网：https://www.nginx.com/
文档：https://docs.nginx.com/
开源文档：https://nginx.org/en/docs/
TCP和UDP负载均衡说明：https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-udp-load-balancer/
TCP健康检测：https://docs.nginx.com/nginx/admin-guide/load-balancer/tcp-health-check//
UDP健康检测：https://docs.nginx.com/nginx/admin-guide/load-balancer/udp-health-check/
ngx_stream_proxy_module模块文档：https://nginx.org/en/docs/stream/ngx_stream_proxy_module.html
ngx_stream_upstream_module模块文档：https://nginx.org/en/docs/stream/ngx_stream_upstream_module.html
ngx_stream_upstream_hc_module模块文档：https://nginx.org/en/docs/stream/ngx_stream_upstream_hc_module.html

注意事项：

• stream配置项和http同级
• 不支持不同域名转发不同Mysql的功能

stream {
    server {
       listen 13306; 
       proxy_connect_timeout 1s;
       proxy_timeout 3s;
       proxy_pass 192.168.1.20:3306;    
    }
}

http {
}

UDP负载均衡

stream {
    upstream dns {
       server 192.168.111.99:10086;
       server 192.168.111.100:10086;
    }

    server {
        listen 192.168.111.98:10086 udp;
        proxy_responses 1; #使用UDP协议时，设置代理服务器响应客户端期望的数据报数。该值作会话终止条件
        proxy_timeout 20s;
        proxy_bind $server_addr:$remote_port;
        proxy_pass dns;
        error_log logs/dns.log;
    }
}

tcp负载均衡配置参数

stream { #定义stream相关的服务；Context:main

    log_format proxy '$remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
   
    access_log  /var/log/nginx/access.log  proxy; 

    upstream backend { #定义后端服务器
        hash $remote_addr consistent; #定义调度算法
        server backend1.example.com:12345 weight=5; #定义具体server
        server 127.0.0.1:12345 max_fails=3 fail_timeout=30s;
        server unix:/tmp/backend3;
    }
    upstream dns { #定义后端服务器
       server 10.0.0.1:53535; #定义具体server
       server dns.example.com:53;
    }
    server { #定义server
        listen 12345; #监听IP:PORT
        proxy_connect_timeout 1s; #连接超时时间
        proxy_timeout 3s; #转发超时时间
        proxy_pass backend; #转发到具体服务器组
    }
    server {
        listen 127.0.0.1:53 udp reuseport;
        proxy_timeout 20s;
        proxy_pass dns;
    }
    server {
        listen [::1]:12345;
        proxy_pass unix:/tmp/stream.socket;
    }
}

负载均衡实例 : Redis

stream {
    upstream redis_server {
        #hash $remote_addr consistent;
        server 172.31.0.28:6379 max_fails=3 fail_timeout=30s;
        server 172.31.0.48:6379 max_fails=3 fail_timeout=30s;
    }
    server {
        listen 172.31.0.18:6379;
        proxy_connect_timeout 3s;
        proxy_timeout 3s;
        proxy_pass redis_server;
    }
}

负载均衡实例: MySQL

stream {
    upstream mysql_server {
        least_conn;
        server 172.31.0.28:3306 max_fails=3 fail_timeout=30s;
    }
    server {
        listen 172.31.0.18:3306;
        proxy_connect_timeout 6s;
        proxy_timeout 15s;
        proxy_pass mysql_server;
    }
}

stream {        
    upstream mysql {
        server 192.168.2.56:3306;  #后端数据库的ip和端口，如果进行了域名解析，直接写域名就好
    }
    server {
        listen 3306;   #如果监听3306，远程登录的时候不用加-p参数
        proxy_connect_timeout 10s;
        proxy_timeout 30s;
        proxy_pass mysql;
    }
}

官方示例分析

nginx监听本地12345端口的TCP数据包，反向代理给stream_backend组，最小连接数方式进行负载均衡。
nginx监听本地53端口的UDP数据包，反向代理给dns_servers组，最小连接数方式进行负载均衡。
nginx监听本地12346端口的TCP数据包，直接反向代理给backend4.example.com:12346

stream {
    upstream stream_backend {
        least_conn;
        server backend1.example.com:12345 weight=5;
        server backend2.example.com:12345 max_fails=2 fail_timeout=30s;
        server backend3.example.com:12345 max_conns=3;
    }
    
    upstream dns_servers {
        least_conn;
        server 192.168.136.130:53;
        server 192.168.136.131:53;
        server 192.168.136.132:53;
    }
    
    server {
        listen        12345;
        proxy_pass    stream_backend;
        proxy_timeout 3s;
        proxy_connect_timeout 1s;
    }
    
    server {
        listen     53 udp;
        proxy_pass dns_servers;
    }
    
    server {
        listen     12346;
        proxy_pass backend4.example.com:12346;
    }
}

mysql代理示例

# ./nginx.conf
...

stream {
    proxy_connect_timeout 3s;
    include stream/*conf;
}

...

# ./stream/mysql.conf
upstream mysql {
    server 192.168.1.100:3306;
}
server {
    listen 3306;
    proxy_connect_timeout 3s;
    proxy_timeout 3s;
    proxy_pass mysql;
}

ssh 代理示例

# ./nginx.conf
...

stream {
    proxy_connect_timeout 3s;
    include stream/*conf;
}

...

# ./stream/ssh.conf
upstream ssh {
    server 0.0.0.0:22;
}
 
server {
    listen 22122;
    proxy_pass ssh;
}