Nginx访问日志和错误日志的拆分(Logstash)
>> from zhuhaiqing.info
input { file { type =>> "nginx-access" path =>> [ "/var/log/nginx/access.log" ] tags =>> [ "nginx","access"] start_position =>> beginning } file { type =>> "nginx-error" path =>> [ "/var/log/nginx/error.log" ] tags =>> [ "nginx","error"] start_position =>> beginning } } filter { if [type] == "nginx-access" { grok{ match =>> ["message","%{IPORHOST:client_ip}\s{1,}\-\s\-\s\[%{HTTPDATE:time}\]\s{1,}\"(?:%{WORD:verb}\s{1,}%{NOTSPACE:request}(?:\s{1,}HTTP/%{NUMBER:http_version})?|-)\" %{NUMBER:response}\s{1,}(?:%{NUMBER:bytes}|-)\s{1,}%{QS:referrer}\s{1,}%{QS:agent}"] } date{ match =>> ["time","dd/MMM/yyyy:HH:mm:ss Z"] target =>> "logdate" } ruby{ code =>> "event.set('logdateunix',event.get('logdate').to_i)" } } else if [type] == "nginx-error" { grok { match =>> [ "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}(%{NUMBER:pid:int}#%{NUMBER}:\s{1,}\*%{NUMBER}|\*%{NUMBER}) %{DATA:err_message}(?:,\s{1,}client:\s{1,}(?<client_ip>%{IP}|%{HOSTNAME}))(?:,\s{1,}server:\s{1,}%{IPORHOST:server})(?:, request: %{QS:request})?(?:, host: %{QS:client_ip})?(?:, referrer: \"%{URI:referrer})?", "message", "(?<time>\d{4}/\d{2}/\d{2}\s{1,}\d{2}:\d{2}:\d{2})\s{1,}\[%{DATA:err_severity}\]\s{1,}%{GREEDYDATA:err_message}" ] } date{ match=>>["time","yyyy/MM/dd HH:mm:ss"] target=>>"logdate" } ruby{ code =>> "event.set('logdateunix',event.get('logdate').to_i)" } } } output{ elasticsearch{ hosts =>> ["192.168.100.10:9200"] index =>> "logstash-nginx-%{+YYYY.MM.dd}" } }