参考文档：
https://blog.csdn.net/loongshawn/article/details/78199977 nginx不转发http header问题解决
https://blog.csdn.net/shootyou/article/details/6335301 使用Nginx转发真实头信息
https://www.jianshu.com/p/6df4092a9a69 nginx处理长连接请求，解决gateway timeout错误
系统依赖在线安装
yum -y install gcc gcc-c++ pcre pcre-devel zlib zlib-devel openssl openssl-devel
获取nginx安装包(http://nginx.org/en/download.html)
wget http://nginx.org/download/nginx-1.16.1.tar.gz tar zxvf nginx-1.16.1.tar.gz tar cd nginx-1.16.1/ ./configure --prefix=/usr/local/nginx --with-http_ssl_module --with-http_sub_module --with-http_realip_module make && make install
需要添加的模块：
--with-http_ssl_module(ssl模块)
--with-http_sub_module(内容替换)
--with-http_realip_module(获取真实IP)
配置nginx系统服务
vim /usr/lib/systemd/system/nginx.service
[Unit] Description=Nginx After=network.target remote-fs.target nss-lookup.target [Service] Type=forking WorkingDirectory=/usr/local/nginx ExecStart=/usr/local/nginx/sbin/nginx ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s QUIT $MAINPID PrivateTmp=true [Install] WantedBy=multi-user.target
相关命令
# 更新系统服务 systemctl daemon-reload # 添加开机自启动 systemctl enable nginx # 启动nginx systemctl start nginx # 关闭nginx systemctl stop nginx # 停止开机自启动 systemctl disable nginx # 查看状态 systemctl status nginx # 重启服务 systemctl restart nginx # 重新加载配置文件(注意：配置文件出错时，刷新不会成功，需要关注log文件输出) /usr/local/nginx/sbin/nginx -s reload
配置nginx主配置文件中增加配置引用，使用配置引入的好处是，避免多个配置混杂。也便于配置切换。
Vi /usr/local/nginx/conf/nginx.conf
# 修改用户，否则日志无法写入硬盘 user root # 修改处理线程数，提升性能,最好与服务器CPU数量一致 worker_processes 2; http { …… …… server { listen 80; …… } #注意添加位置 include conf.d/*.conf; }
在nginx的conf目录下建立conf.d目录
conf/conf.d/test-nginx.conf
# 分流配置，根据转发地址修改 upstream asnp_proxy { # 如果失败两次，则60秒后重试 # server 172.21.32.32:5888 max_fails=2 fail_timeout=60s; # server 172.21.32.32:6888 max_fails=2 fail_timeout=60s; server 172.21.100.72:8080; } #日志格式设定，无需修改 #$remote_addr与$http_x_forwarded_for用以记录客户端的ip地址； #$remote_user：用来记录客户端用户名称； #$time_local： 用来记录访问时间与时区； #$request： 用来记录请求的url与http协议； #$status： 用来记录请求状态；成功是200,错误是500 #$body_bytes_sent ：记录发送给客户端主体内容大小； #$http_referer：用来记录从那个页面链接访问过来的； #$http_user_agent：记录客户浏览器的相关信息； #$request_time ： 整个请求的总时间 #$upstream_response_time：请求过程中，upstream响应时间 #$bytes_sent :客户端发送的字节数 #$request_length：客户端请求的长度 #$upstream_status：upstream状态 #$upstream_addr ：后台upstream的地址，即真正提供服务的主机地址 log_format asnp_main '{ "@timestamp": "$time_iso8601", ' '"access_time": "$time_iso8601", ' '"remote_addr": "$remote_addr", ' '"remote_user": "$remote_user", ' '"adt_host": "$host", ' '"request": "$request", ' '"uri": "$uri", ' '"request_method": "$request_method", ' '"status": "$status", ' '"request_time": "$request_time", ' '"upstream_response_time": "$upstream_response_time", ' '"request_length": "$request_length", ' '"bytes_sent": "$bytes_sent", ' '"upstream_addr": "$upstream_addr", ' '"upstream_status": "$upstream_status", ' '"http_referrer": "$http_referer", ' '"http_x_forwarded_for": "$http_x_forwarded_for", ' '"http_user_agent": "$http_user_agent" ' '}'; # 限流配置，如有限流需要则开启,开启后需要在loaction中增加对应配置 # limit_req_zone $uri zone=db_access:20m rate=1r/s; server { # 对外的代理端口，根据需要修改 listen 80; # 代理服务器名称，填写本机IP即可 server_name 172.18.100.72; # 日志配置，可以按各种频率输出，目前是按月，也可以按天 if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})") { set $year $1; set $month $2; set $day $3; set $hour $4; set $minutes $5; set $seconds $6; } # 日志输出路径配置 access_log logs/asnp_access-$year-$month.log asnp_main; error_log logs/asnp_error.log; # 开放http访问 location / { proxy_pass http://asnp_proxy/; # 设置IP proxy_set_header X-Real-IP $remote_addr; #后端的Web服务器可以通过X-Forwarded-For获取用户真实IP proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # 连接后端服务器超时时间(秒) proxy_connect_timeout 30; # 将域名代理过去 proxy_set_header Host $host:$server_port; real_ip_header X-Forwarded-For; real_ip_recursive on; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_headers_hash_max_size 512; proxy_headers_hash_bucket_size 128; } }
# 限流配置，如有限流需要则开启,开启后需要在loaction中增加对应配置 # limit_req_zone $uri zone=db_access:20m rate=1r/s; http{ .... # 限流测试样例，指定url进行限流 # location /http/mytest2 { # proxy_pass http://aesb_proxy/http/mytest2; # # # 选择前面的限流规则 # limit_req zone=db_access burst=5; # } }
创建一个数据库，编码选择UTF8MB4.数据库名任意，建议叫logstash。用于存储mysql日志。
DROP TABLE IF EXISTS `t_nginxlog`; CREATE TABLE `t_nginxlog` ( `id` int(11) NOT NULL AUTO_INCREMENT, `stand_time` timestamp NULL DEFAULT NULL, `access_time` timestamp NULL DEFAULT NULL, `remote_addr` varchar(50) DEFAULT NULL, `remote_user` varchar(50) DEFAULT NULL, `adt_host` varchar(50) DEFAULT NULL, `request` varchar(200) DEFAULT NULL, `uri` varchar(500) DEFAULT NULL, `request_method` varchar(20) DEFAULT NULL, `status` varchar(20) DEFAULT NULL, `request_time` varchar(20) DEFAULT NULL, `upstream_response_time` varchar(20) DEFAULT NULL, `request_length` varchar(20) DEFAULT NULL, `bytes_sent` varchar(20) DEFAULT NULL, `upstream_addr` varchar(50) DEFAULT NULL, `upstream_status` varchar(20) DEFAULT NULL, `http_referrer` varchar(255) DEFAULT NULL, `http_x_forwarded_for` varchar(500) DEFAULT NULL, `http_user_agent` varchar(500) DEFAULT NULL, `insert_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;
安装logstash进行日志采集（待续）
下载安装包
wget https://mirrors.huaweicloud.com/logstash/7.8.0/logstash-7.8.0.tar.gz tar zxvf logstash-7.8.0.tar.gz # 指定jdbc扩展包，进行安装否则不能访问数据库，配置文件中需要指定jdbc驱动路径。 bin/logstash-plugin install file:///home/logstash-output-jdbc.zip
制作logstash日志采集任务文件
# Sample Logstash configuration for creating a simple # Beats -> Logstash -> Elasticsearch pipeline. input { file { path => ["/usr/local/nginx/logs/asnp_access*.log"] start_position => "beginning" codec => "json" } } filter { date { match => [ "access_time", "YYYY-MM-dd'T'HH:mm:ssZ"] target => "access_time" } # add 8 hour ruby { code => "event.set('access_time', event.get('access_time').time.localtime + 8*60*60)" } } output { stdout { # codec => json codec => rubydebug } jdbc { driver_jar_path => "/opt/logstash/mysql-connector-java-5.1.42.jar" driver_class => "com.mysql.jdbc.Driver" connection_string => "jdbc:mysql://172.18.100.70:3306/logstash?user=root&password=root&useUnicode=true&characterEncoding=UTF-8&useSSL=false" statement => [ "INSERT INTO t_nginxlog (stand_time,access_time,remote_addr,remote_user,adt_host,request,uri,request_method,status,request_time,upstream_response_time,request_length,bytes_sent,upstream_addr,upstream_status ,http_referrer,http_x_forwarded_for,http_user_agent) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)","@timestamp","access_time","remote_addr","remote_user","adt_host","request","uri","request_method","status","request_time","upstre am_response_time","request_length","bytes_sent","upstream_addr","upstream_status","http_referrer","http_x_forwarded_for","http_user_agent"]} }
启动logstash进行日志采集
# 后台启动，指定对应的配置文件 nohup ./logstash-7.8.0/bin/logstash -f asnp-log.conf --config.reload.automatic &
安装granafa
https://mirrors.huaweicloud.com/grafana/7.1.1/grafana-7.1.1.linux-amd64.tar.gz
直接解压启动即可，默认端口为3000，用户为admin/admin
