logstash收集nginx访问日志

本文大部分参考及引用：https://www.cnblogs.com/Dev0ps/p/9313418.html

1.安装nginx

参考：https://www.cnblogs.com/fuanyu/p/14601345.html

目前修改nginx.conf 

官方文档：http://nginx.org/en/docs/http/ngx_http_log_module.html#log_format
#修改配置文件的日志格式：
vi  /usr/local/nginx/conf/nginx.conf 
#在http模块中添加
          log_format json '{"@timestamp":"$time_iso8601",'
                           '"@version":"1",'
                           '"client":"$remote_addr",'
                           '"url":"$uri",'
                           '"status":"$status",'
                           '"domain":"$host",'
                           '"host":"$server_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"referer": "$http_referer",'
                           '"ua": "$http_user_agent"'
               '}';
#在server模块中添加
access_log  /usr/local/nginx/logs/access.log  json;

保存后，启动 

/usr/local/nginx/sbin/nginx

完整的nginx.conf文件如

user root;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  otice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
   # include       ip.black;
    default_type  application/octet-stream;

     log_format access_json '{"@timestamp":"$time_iso8601",'
                           '"host":"$server_addr",'
                           '"clientip":"$remote_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"upstreamtime":"$upstream_response_time",'
                           '"upstreamhost":"$upstream_addr",'
                           '"http_host":"$host",'
                           '"url":"$uri",'
                           '"domain":"$host",'
                           '"xff":"$http_x_forwarded_for",'
                           '"referer":"$http_referer",'
                           '"status":"$status"}';
    access_log  /usr/local/nginx/logs/access.log  access_json;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

  #定义一个名为allips的limit_req_zone用来存储session，大小是10M内存，
  #以$binary_remote_addr 为key,限制平均每秒的请求为5个，
  #1M能存储16000个状态，rete的值必须为整数，
  #如果限制两秒钟一个请求，可以设置成30r/m
  limit_req_zone $binary_remote_addr zone=allips:10m rate=5r/s;
    #gzip  on;

        upstream tomcat_server{
                server 172.16.38.225:18001 weight=1;
                server 172.16.38.226:18001 weight=1;
        }
        upstream socket_server{
                server 172.16.38.225:8099 weight=1;
                server 172.16.38.226:8099 weight=1;
        }
        server {
        listen       80;
        server_name  localhost;
                        location / {
                                # root /home/oracle/dev_tools/server/apache-tomcat-6.0.44/webapps/;

                                # HTTP代理模块 proxy，主要是用来转发请求到其他服务器
                                # 如果后端服务器返回502，504，执行超时等错误，自动将请求转发到upstream负载均衡池中的另一台服务器，实现failover。
                                # WebScoket Support
                                proxy_headers_hash_max_size 51200;
                                proxy_headers_hash_bucket_size 6400;

                                proxy_redirect  off;
                                proxy_set_header X-Real-IP $remote_addr;
                                proxy_set_header Host $host;
                                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

                                proxy_http_version 1.1;
                                proxy_set_header Upgrade $http_upgrade;
                                proxy_set_header Connection "upgrade";


                                proxy_next_upstream http_502 http_504 error timeout invalid_header;
                                # 变量$host等于客户端请求头中的Host值。
                                proxy_set_header Host $host;
                                #后端的web服务器可以通过X-Forwarded-For获取真实的IP地址，$remote_addr客户端的ip地址
                                proxy_set_header X-Forwarded-For $remote_addr;
                                proxy_pass http://socket_server;

                                #限制每ip每秒不超过20个请求，漏桶数burst为5
                               #brust的意思就是，如果第1秒、2,3,4秒请求为19个，
                                 #第5秒的请求为25个是被允许的。
                                 #但是如果你第1秒就25个请求，第2秒超过20的请求返回503错误。
                                 #nodelay，如果不设置该选项，严格使用平均速率限制请求数，
                                 #第1秒25个请求时，5个请求放到第2秒执行，
                                #设置nodelay，25个请求将在第1秒执行。
                                limit_req zone=allips burst=5 nodelay;
                        }

                }

    

}

用浏览器请求: http://172.16.38.225

会看到access.log的输入日记

2.编写logstash

vi /usr/local/app/logstash-6.2.4/config/logstash-nginx.conf

input {
    
       file {
       path => "/usr/local/nginx/logs/access.log"
       codec => json
       start_position => "beginning"
       type => "nginx-log"
    }
}
output {
    
       if [type] == "nginx-log"{
        elasticsearch {
           hosts => ["172.16.38.225:9200"]
           index => "nginx-log-%{+YYYY.MM.dd}"
        }
    }
}

然后再启动

./logstash -f  ../config/logstash-nginx.conf &

 

3|0kibana添加nginx日志

首先在es插件中我们能看到nginx-log的索引

 设置kibana