logstash收集ngx日志

 

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41

if [type] =~ "ngx-" {         #去掉重复的行         mutate {             add_field => {"line_message" => "%{message} %{offset}"}           }         ruby {         code => "         require 'digest/md5';         event.set('computed_id', Digest::MD5.hexdigest(event.get('line_message')))             "         }         #匹配nginx日志         grok {              match => { "message" => "%{IPORHOST:clientip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:timestamp}\] \"(?:%{WORD:method} %{NOTSPACE:request}(?: %{URIPROTO:proto}/%{NUMBER:httpversion})?|%{DATA:rawrequest})\" %{NUMBER:status} (?:%{NUMBER:size}|-) \"(?:%{DATA:referrer}|-)\" \"(?:%{DATA:agent}|-)\" \"(%{DATA:xforwardedfor}|-)\" \"(?:%{DATA:domain}|-)\" \"%{NUMBER:server_port}\" %{NUMBER:reqtime} %{DATA:forward_ip}" }              remove_field => ["source","host","message","forward_ip","domain","beat.name","remote_user"]         }         #设置日期格式         date {             match => ["timestamp", "dd/MMM/YYYY:HH:mm:ss +0800"]             target => "@timestamp"             "locale" => "en"             timezone => "UTC"             remove_field => ["timestamp"]         }         #设置IP地址         geoip {             source => "clientip"             target => "geoip"         }         #删除[geoip][ip]以192.100.10.的IP地址         if [geoip][ip] =~ "192.100.10." {             drop {}         }         #删除行信息         mutate {             remove_field => ["line_message"]         }         useragent {             source => "agent"         }     }