ELK系统分析nginx日志

一、nginx

nginx 服务器日志的log_format格式：

    log_format  main '$remote_addr - $remote_user [$time_local] "$request" '
              '$status $body_bytes_sent "$http_referer" '
              '"$http_user_agent"  $request_time';
    access_log  logs/access.log  main;

nginx日志文件其中一行：

10.6.97.167 - - [20/Dec/2018:16:43:20 +0800] "GET /static/image/common/scrolltop.png HTTP/1.1" 304 0 "http://10.6.191.183/data/cache/style_1_common.css?JT9" "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0"  0.000

二、配置logstash

[root@localhost ~]# cat /usr/local/logstash/config/etc/nginx.conf 
input {
    file {
        path => [ "/usr/local/nginx/logs/access.log" ]
        start_position => "beginning"
        ignore_older => 0
    }
}

filter {
    grok {
        patterns_dir => [ "/usr/local/logstash/patterns" ]
        match => { "message" => "%{NGINXACCESS}" }
        
    }

    date {
      match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]

    }

}
output {
    elasticsearch {
        hosts => ["10.6.191.181:9200"]
        index => "logstash-nginx-access-%{+YYYY.MM.dd}"
    }
    stdout {codec => rubydebug}
}

input {
    file {
        path => [ "/usr/local/nginx/logs/access.log" ]
        start_position => "beginning"
        ignore_older => 0
    }
}

filter {
    grok {
        patterns_dir => [ "/usr/local/logstash/patterns" ]
        match => { "message" => "%{NGINXACCESS}" }

    }

    geoip {
      source => "clientip"
      target => "geoip"
      database => "/usr/local/logstash/GeoLiteCity.dat"
      add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
      add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
    }

    mutate {
      convert => [ "[geoip][coordinates]", "float" ]
      convert => [ "response","integer" ]
      convert => [ "bytes","integer" ]
      replace => { "type" => "nginx_access" }
      remove_field => "message"
    }

    date {
      match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]

    }

}
output {
    elasticsearch {
        hosts => ["10.6.191.181:9200"]
        index => "logstash-nginx-access-%{+YYYY.MM.dd}"
    }
    stdout {codec => rubydebug}
}

配置grok正则格式匹配message

[root@localhost ~]# cat /usr/local/logstash/patterns/nginx        
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:verb} 
%{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:bytes}|-) %{QS:referrer} %{QS:agent}