生产环境Nginx部署案例

常规化部署Nginx手册

前期工作

• 从公司下载下载nginx稳定版本1.20.2 和相关Nginx插件
• ningx作为代理服务器,服务器的文件打开数设置为最大65535

[root@localhost ~]# cat >> /etc/security/limits.conf <<EOF
* soft nproc 65535
* hard nproc 65535
* soft nofile 65535
* hard nofile 65535
EOF
[root@localhost ~]# echo "ulimit -SH 65535" >> /etc/rc.local
[root@localhost ~]# echo "ulimit -n 65535 "  >> /etc/profile
[root@localhost ~]# source /etc/profile
[root@localhost ~]# ulimit -a

• 注意事项：在复制时注意复制时有自动换行符，导致缺失数据。

[root@localhost ~]# cd /usr/local/src/
[root@localhost src]#  wget http://10.130.36.117/nginx/nginx-1.20.2.tar.gz
[root@localhost src]#  wget http://10.130.36.117/nginx/plug-in/naxsi-0.55.3.tar.gz
[root@localhost src]#  wget http://10.130.36.117/nginx/plug-in/nginx-code-gcc.tar.gz
[root@localhost src]#  wget http://10.130.36.117/nginx/plug-in/nginx-goodies-nginx-sticky-module-ng.zip
[root@localhost src]#  wget http://10.130.36.117/nginx/plug-in/nginx_upstream_check_module-master.zip
[root@localhost src]#  wget http://10.130.36.117/Deploymentpackage/v0.10.13.tar.gz 
[root@localhost src]#  wget http://10.130.36.117/Deploymentpackage/LuaJIT-2.0.4.tar.gz

• 安装高版本openssl
• 参考升级部署ssl文档：http://10.130.1.65:8090/pages/viewpage.action?pageId=42369582

[root@localhost src]# wget http://10.130.36.117/OpenBSD/OpenSSH/portable/openssh-8.6p1.tar.gz
[root@localhost src]# wget http://10.130.36.117/OpenBSD/openssl/source/openssl-1.1.1g.tar.gz
[root@localhost src]# wget http://10.130.36.117/OpenBSD/zlib/zlib-1.2.11.tar.gz
[root@localhost src]# wget http://10.130.36.117/perl/perl-5.10.1.tar.bz2

• 由于openssh需要依赖openssl，所以openssh也一并需要进行升级

#解压升级包
[root@localhost src]# tar -xvf openssh-8.6p1.tar.gz
[root@localhost src]# tar -xvf openssl-1.1.1g.tar.gz
[root@localhost src]# tar -xvf zlib-1.2.11.tar.gz
[root@localhost src]# yum install -y gcc gcc-c++ glibc make autoconf openssl openssl-devel pcre-devel  pam-devel 
 
#安装zlib
[root@localhost src]# cd zlib-1.2.11/
[root@localhost zlib-1.2.11]# ./configure --prefix=/usr/local/zlib
[root@localhost zlib-1.2.11]# make -j 4 && make install
 
#编译安装openssl
[root@localhost src]# cd /usr/local/src/openssl-1.1.1g/
[root@localhost openssl-1.1.1g]# ./config --prefix=/usr/local/ssl -d shared
[root@localhost openssl-1.1.1g]# make -j 4 && make install
[root@localhost openssl-1.1.1g]# echo '/usr/local/ssl/lib' >> /etc/ld.so.conf
[root@localhost openssl-1.1.1g]# ldconfig -v
 
#centos5.x操作系统等系列默认Perl 版本v5.8.8 OpenSSL 需要 v5.10.0版本支持。
 
#perl
[root@localhost openssl-1.1.1g]# cd ../
[root@portal_node_1 src]# tar xvf perl-5.10.1.tar.bz2 
[root@localhost src]# cd perl-5.10.1
[root@localhost perl-5.10.1]#./Configure -des -Dprefix=/usr/local/perl
[root@localhost perl-5.10.1]# make -j 4  && make install
[root@localhost perl-5.10.1]# mv /usr/bin/perl /usr/bin/perl.bak
[root@localhost perl-5.10.1]# ln -s /usr/local/perl/bin/perl /usr/bin/perl
 
#安装openssh
[root@localhost openssh-8.6p1]# cd /usr/local/src/openssh-8.6p1/
[root@localhost openssh-8.6p1]# vim version.h
#define SSH_VERSION "OpenSSH_ttxs" ##修改此处
[root@localhost openssh-8.6p1]# ./configure --prefix=/usr --sysconfdir=/etc/ssh --with-zlib=/usr/local/zlib --with-ssl-dir=/usr/local/ssl --with-privsep-path=/var/lib/sshd
[root@localhost openssh-8.6p1]# make -j 4 && make install
 
#sshd_config文件修改
[root@localhost openssh-8.6p1]# vim /etc/ssh/sshd_config //修改sshd_config 配置文件
[root@localhost openssh-8.6p1]# sed -i s/"#PermitRootLogin yes"/"PermitRootLogin yes"/g /etc/ssh/sshd_config
[root@localhost openssh-8.6p1]# grep "^PermitRootLogin" /etc/ssh/sshd_config
PermitRootLogin yes
[root@localhost openssh-8.6p1]# sed -i s/"UsePAM yes"/"#UsePAM yes"/g /etc/ssh/sshd_config
[root@localhost openssh-8.6p1]# grep "UsePAM" /etc/ssh/sshd_config
 #UsePAM yes
[root@localhost openssh-8.6p1]# sed -i s/"GSSAPIAuthentication yes"/"#GSSAPIAuthentication yes"/g /etc/ssh/sshd_config
[root@localhost openssh-8.6p1]# grep "GSSAPIAuthentication" /etc/ssh/sshd_config
#GSSAPIAuthentication no
#GSSAPIAuthentication yes
[root@localhost openssh-8.6p1]# sed -i s/"GSSAPICleanupCredentials yes"/"#GSSAPICleanupCredentials yes"/g /etc/ssh/sshd_config
[root@localhost openssh-8.3p1]# grep "GSSAPICleanupCredentials" /etc/ssh/sshd_config
#GSSAPICleanupCredentials yes
#GSSAPICleanupCredentials yes
[root@localhost openssh-8.3p1]# chmod 600 /etc/ssh/ssh_host_dsa_key /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_ed25519_key /etc/ssh/ssh_host_key /etc/ssh/ssh_host_rsa_key
 
#Centos6系统，启动sshd,并查看版本信息
[root@localhost openssh-8.6p1]# service sshd restart
[root@localhost openssh-8.6p1]# ssh -V
OpenSSH_ttxsp1, OpenSSL 1.1.1g  21 Apr 2020
 
#Centos7系统，启动sshd,并查看版本信息
[root@localhost openssh-8.6p1]# cp -a contrib/redhat/sshd.init /etc/init.d/sshd
[root@localhost openssh-8.6p1]# chmod +x /etc/init.d/sshd
[root@localhost openssh-8.6p1]# chkconfig --add sshd
[root@localhost openssh-8.6p1]# chmod +x /etc/init.d/sshd
[root@localhost openssh-8.6p1]# systemctl enable sshd
[root@localhost openssh-8.6p1]# mv /usr/lib/systemd/system/sshd.service /usr/local/
[root@localhost openssh-8.6p1]# chkconfig sshd on
注意：正在将请求转发到“systemctl enable sshd.socket”。
Created symlink from /etc/systemd/system/sockets.target.wants/sshd.socket to /usr/lib/systemd/system/sshd.socket.
[root@localhost openssh-8.6p1]# service sshd restart
Restarting sshd (via systemctl): [ 确定 ]
[root@localhost openssh-8.6p1]# systemctl status sshd
● sshd.service - SYSV: OpenSSH server daemon
Loaded: loaded (/etc/rc.d/init.d/sshd; bad; vendor preset: enabled)
Active: active (running) since 五 2020-08-07 18:28:28 CST; 2s ago

编译Nginx

• nginx插件打补丁,并编译安装,特殊说明openssl1.1.1版本后该功能就不支持。

[root@localhost openssh-8.6p1]# cd /usr/local/src
[root@localhost src]# unzip  nginx-goodies-nginx-sticky-module-ng.zip   
[root@localhost src]# tar xvf naxsi-0.55.3.tar.gz
[root@localhost src]# unzip  nginx_upstream_check_module-master.zip
[root@localhost src]# tar xvf nginx-code-gcc.tar.gz
[root@localhost src]# tar xvf nginx-1.20.2.tar.gz 
[root@localhost src]# tar xvf v0.10.13.tar.gz 
[root@localhost src]# tar -zxvf  LuaJIT-2.0.4.tar.gz
[root@localhost src]# cd LuaJIT-2.0.4
[root@localhost LuaJIT-2.0.4]# make -j 4 &&  make install
[root@localhost LuaJIT-2.0.4]# vim /etc/profile  #文件末尾加入环境变量
export LUAJIT_INC=/usr/local/include/luajit-2.0  #增加luajit环境变量
export LUAJIT_LIB=/usr/local/LuaJIT/lib
[root@localhost LuaJIT-2.0.4]# source /etc/profile
[root@localhost LuaJIT-2.0.4]# ln -s /usr/local/lib/libluajit-5.1.so.2  /lib64/
[root@localhost nginx-1.20.2]# cd /usr/local/src/nginx-1.20.2
 
#对nginx_upstream_check_module插件打补丁
[root@localhost nginx-1.20.2]# patch -p1 <   /usr/local/src/nginx_upstream_check_module-master/check_1.14.0+.patch 
patching file src/http/modules/ngx_http_upstream_hash_module.c
Hunk #2 succeeded at 241 (offset 3 lines).
Hunk #3 succeeded at 571 (offset 6 lines).
patching file src/http/modules/ngx_http_upstream_ip_hash_module.c
Hunk #2 succeeded at 211 (offset 3 lines).
patching file src/http/modules/ngx_http_upstream_least_conn_module.c
patching file src/http/ngx_http_upstream_round_robin.c
Hunk #1 succeeded at 9 with fuzz 2.
Hunk #2 succeeded at 107 (offset 6 lines).
Hunk #3 succeeded at 186 (offset 12 lines).
Hunk #4 succeeded at 263 (offset 13 lines).
Hunk #5 succeeded at 383 (offset 14 lines).
Hunk #6 succeeded at 420 (offset 14 lines).
Hunk #7 succeeded at 488 (offset 14 lines).
Hunk #8 succeeded at 588 (offset 14 lines).
patching file src/http/ngx_http_upstream_round_robin.h
 
[root@localhost nginx-1.20.2]# ./configure --prefix=/usr/local/nginx --with-http_v2_module --with-http_stub_status_module --with-http_ssl_module --add-module=/usr/local/src/nginx-code-gcc/ngx_devel_kit-0.3.0  --add-module=/usr/local/src/lua-nginx-module-0.10.13/ --add-module=/usr/local/src/nginx-goodies-nginx-sticky-module-ng-08a395c66e42 --add-module=/usr/local/src/nginx_upstream_check_module-master --add-module=/usr/local/src/naxsi-0.55.3/naxsi_src/ --with-openssl=/usr/local/src/openssl-1.1.1g --with-stream 
 
[root@localhost nginx-1.20.2]# make -j 4
[root@localhost nginx-1.20.2]# make install
 
#查看编译信息
[root@localhost sbin]# cd /usr/local/nginx/sbin
[root@localhost sbin]# ./nginx -V
nginx version: nginx/1.20.2
built by gcc 4.4.7 20120313 (Red Hat 4.4.7-23) (GCC) 
built with OpenSSL 1.0.1e-fips 11 Feb 2013
TLS SNI support enabled
configure arguments: --prefix=/usr/local/nginx --with-http_v2_module --with-http_stub_status_module --with-http_ssl_module --add-module=/usr/local/src/nginx-code-gcc/ngx_devel_kit-0.3.0  --add-module=/usr/local/src/lua-nginx-module-0.10.13/ --add-module=/usr/local/src/nginx-goodies-nginx-sticky-module-ng-08a395c66e42 --add-module=/usr/local/src/nginx_upstream_check_module-master --add-module=/usr/local/src/naxsi-0.55.3/naxsi_src/ --with-openssl=/usr/local/src/openssl-1.1.1g --with-stream 

Nginx配置

• 以下相关配置文件，对应提供模板进行参考,请根据实际的进行修改和调整。

• 主配置文件

[root@localhost conf]# vim nginx.conf
user  root;
worker_processes  4;
error_log  logs/error.log;
pid        sbin/nginx.pid;
 
events {
    use epoll;
    worker_connections  65535;
}
 
http {
    include       mime.types;
    default_type  application/octet-stream;
 
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for" "$upstream_addr"';
     map $time_iso8601 $logdate{
        '~^(?<ymd>\d{4}-\d{2}-\d{2})' $ymd;
        default 'date-not-found';
    }
    access_log  logs/access-$logdate.log  main;
    sendfile        on;
    keepalive_timeout  65;
    fastcgi_buffers 8 128k;
    proxy_next_upstream_tries 3;
    proxy_connect_timeout 100s;
    proxy_send_timeout 100s;
    proxy_read_timeout 100s;
    fastcgi_connect_timeout 300;
    fastcgi_send_timeout 300;
    fastcgi_read_timeout 300;
    client_max_body_size 200m;
    server_tokens  off;
    #add_header Set-Cookie "HttpOnly";
    #add_header Set-Cookie "Secure";
    #add_header X-Frame-Options SAMEORIGIN;
    include    vhosts/local_upstream.conf;
    include    vhosts/server.conf;
}
 
#创建vhosts目录
[root@localhost conf]# mkdir vhosts
 
#vhosts目录文件
[root@localhost vhosts]# vim server.conf 
server {
    listen 80 default_server;
    server_name  ccod.com;
    include    vhosts/local.conf;
    error_page  404              /error.html;
    error_page   500 502 503 504  /error.html;
    location = /error.html {
        root   html;
    }
}
 
#正向代理，用于非公网访问,使用yum安装
server{
        listen 80;
        server_name    mirrors.aliyun.com;
        location ~ ^/*{
                 proxy_redirect off;
                 proxy_set_header Host $host;
                 proxy_set_header X-Forwarded-Host $host;
                 proxy_set_header X-Forwarded-Server $host;
                 proxy_set_header X-Real-IP $remote_addr;
                 proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                 proxy_buffering off;
                 chunked_transfer_encoding       off;
                 proxy_pass     http://mirrors.aliyun.com;
                 client_max_body_size 512m;
        }
    }
 
server {
    listen 443 ssl http2;
    ssl_certificate /usr/local/nginx/conf/crt/test.com.pem;
    ssl_certificate_key /usr/local/nginx/conf/crt/test.com.key;
    ssl_session_timeout 1d;
    ssl_session_cache shared:MozSSL:100m;
    ssl_session_tickets off;
    #ssl_dhparam /path/to/dhparam;
 
    # intermediate configuration
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;
    ssl_prefer_server_ciphers off;
 
    # HSTS (ngx_http_headers_module is required) (63072000 seconds)
    add_header Strict-Transport-Security "max-age=63072000" always;
 
    # OCSP stapling
    ssl_stapling on;
    ssl_stapling_verify on;
 
    # verify chain of trust of OCSP response using Root CA and Intermediate certs
    #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates;
 
    server_name  *.ccod.com;
    include    vhosts/local.conf;
    error_page  404              /error.html;
    error_page   500 502 503 504  /error.html;
    location = /error.html {
        root   html;
    }
}
 
[root@localhost vhosts]# vim local.conf 
      location /qnstatus{
                  check_status;
                  access_log off;
        }
 
      location /test2/{
        proxy_pass      http://test2;
        proxy_set_header Host $host;
    }
   
[root@localhost vhosts]# vim   local_upstream.conf 
       upstream        gls{
                sticky;
                server  xxx.xxx.com:80;
                check interval=3000 rise=3 fall=5 timeout=1000 type=tcp;  #健康探测
        }
        
 
 

上传证书文件并启动服务验证

• 上传证书

[root@localhost vhosts]# cd /usr/local/nginx/conf/
[root@localhost conf]# wget http://10.130.36.117/Deploymentpackage/crt.tar.gz 
[root@localhost conf]# tar xvf crt.tar.gz 
 
[root@localhost conf]# cd /usr/local/nginx/sbin
[root@localhost sbin]# ./nginx  -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@localhost sbin]# ./nginx 

Nginx模板

• 下载可以直接配置文件进行修改并使用 http://10.130.36.117/nginx/plug-in/nginx.tar.gz

• 健康探测访问

Nginx限流

• Nginx官方版本限制IP的连接和并发分别有两个模块

  • limit_req_zone 用来限制单位时间内的请求数，即速率限制,采用的漏桶算法 “leaky bucket”，作用域：http、server、location。
  • limit_req_conn 用来限制同一时间连接数，即并发限制,作用域：http、server、location，用来限制单个IP的请求数。并非所有的连接都被计数。只有在服务器处理了请求并且已经读取了整个请求头时，连接才被计数。
  • limit_req 用来控制流量下载速度作用域：http、server、location。

• limit_req_zone案例说明

关键字说明：
   1、$binary_remote_addr 表示通过remote_addr这个标识来做限制，“binary_”的目的是缩写内存占用量，是限制同一客户端ip地址。
   2、one=one:10m表示生成一个大小为10M，名字为one的内存区域，用来存储访问的频次信息。
   3、rate=1000r/s表示允许相同标识的客户端的访问频次，这里限制的是每秒1000次
   4、limit_req zone=one burst=5 nodelay; #zone=one 设置使用哪个配置区域来做限制，与上面limit_req_zone 里的name对应。
   5、burst=5，重点说明一下这个配置，burst爆发的意思，这个配置的意思是设置一个大小为5的缓冲区当有大量请求（爆发）过来时，超过了访问频次限制的请求可以先放到这个缓冲区内。
	6、nodelay，如果设置，超过访问频次而且缓冲区也满了的时候就会直接返回503，如果没有设置，则所有请求会等待排队。
 
http {
    limit_req_zone $binary_remote_addr zone=one:10m rate=1000r/s;
    server {
        location /group1/ {
        limit_req zone=one burst=5 nodelay;
        }
    }      
}

• limit_conn结合limit_req案例说明

关键字说明：
   1、limit_conn限制单个ip并发的请求数量
   2、limit_rate限制网络的速度
   3、limit_rate_after限制超过多少了开始限速
   
location /group1/ {
     limit_conn 10;
     limit_rate_after 500k;
     limit_rate 100k
}

Nginx限制访问

• allow和deny配置段为： http、server、location

• 案例

  • 从上到下的顺序，类似iptables。匹配到了便跳出。如上的例子先禁止了192.168.128.10，接下来允许了2个网段，最后未匹配的IP全部禁止访问. 在实际生产环境中，我们也会使用nginx 的geo模块配合使用

location / {
deny  192.168.128.10;
allow 192.168.128.0/24;
allow 10.100.0.0/24;
deny  all;
}

Tengine 根据时间段来限流

• 需要重编译阿里云开发 Tengine 程序包，该版本是基于nginx-1.18版本更新的
• 编译操作和nginx是一致的
• 相关配置限流配置

[root@portal_node_1 conf]# vim nginx.conf #在http块中添加如下配置,其中rate值可以使用变量的方式来设置
    limit_req_zone $binary_remote_addr $request_uri zone=four:3m rate=$limit_count;
 
#下面案例是针对gls门户访问，在工作时间每秒限制一次访问，非工作时间每秒10000次的访问
[root@portal_node_1 conf]# vim vhosts/local_location.conf 
        location /gls/{
        if ($hour  ~*  "08|09|10|11|12|13|14|15|16") {
           set $limit_count "1r/s";
        }
 
        if ($hour  ~* "17|18|19|20|21|22|23|00|01|02|03|04|05|06|07") {
            set $limit_count "10000r/s";
        }
 
        limit_req zone=four burst=3 forbid_action=@test2;
        proxy_pass      http://gls;
        proxy_set_header Host $host;
       }

• 根据时间段限制流量访问用于下载录音

    location /group1 {
        if ($hour  ~*  "08|09|10|11|12|13|14|15|16") {
           set $bandwidth "300k";
           set $exceeding "50k";
        }
 
        if ($hour  ~* "17|18|19|20|21|22|23|00|01|02|03|04|05|06|07") {
            set $bandwidth "3000k";
            set $exceeding "300k";
        }
 
        limit_rate_after $bandwidth;
        limit_rate $exceeding;
        proxy_pass   http://server_group1;
        add_header Access-Control-Allow-Origin *;
        add_header Access-Control-Allow-Headers X-Requested-With;
        add_header Access-Control-Allow-Methods GET,POST;
        add_header   Content-Type application/X-download;
    }

nginx获取自定义消息头

• nginx 自定义少消息头判断

#在客户端请求标头字段中启用或禁用下划线。禁止使用下划线时，名称中包含下划线的请求标头字段将被标记为无效，并受[ignore_invalid_headers](http://nginx.org/en/docs/http/ngx_http_core_module.html#ignore_invalid_headers)指令的约束 。
underscores_in_headers on;

• 获取请求头信息
  • 比如我们自定义header为X-Real-IP,通过第二个nginx获取该header时需要这样: $http_test_version; (一律采用小写，而且前面多了个http_，且横杠test-version变成了下划线test_version)
• 在nginx配置文件只使用if $http_test_version 判断转发不同的代理。