Logstash深入收集Nginx日志

Logstash深入收集Nginx日志

安装nginx

[root@elkstack03 ~]# yum install -y nginx


## 主配置文件
[root@elkstack03 ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;
}

## 子配置文件
[root@elkstack03 ~]# vim /etc/nginx/conf.d/www.conf
  
server{
        listen 80;
        server_name _;
        root /code;
        index index.html;
}

[root@elkstack03 ~]# mkdir /code
[root@elkstack03 ~]# echo 'test nginx' > /code/index.html
[root@elkstack03 ~]# systemctl start nginx

将nginx日志改成Json格式

之前我们讲了tomcat日志，在企业中，修改格式需要与开发商量，但是nginx我们不需要，如果需要原来的格式日志，我们可以将日志输出两份，一份 main格式，一份Json格式

http{
		...
	log_format json '{"@timestamp":"$time_iso8601",'
       	'"host":"$server_addr",'
       	'"ipaddr":"$remote_addr",'
       	'"login_user":"$remote_user",'
       	'"size":$body_bytes_sent,'
       	'"responsetime":$request_time,'
       	'"upstreamtime":"$upstream_response_time",'
       	'"upstreamhost":"$upstream_addr",'
       	'"http_host":"$host",'
       	'"url":"$uri",'
       	'"domain":"$host",'
       	'"xff":"$http_x_forwarded_for",'
       	'"referer":"$http_referer",'
       	'"status":"$status"}';
		...
}


[root@elkstack03 conf.d]# vim www.conf 
server{
        listen 80;
        server_name www.zls.com;
        root /code;
        index index.html;
        access_log  /var/log/nginx/www.zls.com_access_json.log  json;
}

[root@elkstack03 conf.d]# cat /etc/nginx/conf.d/blog.conf 
server{
	listen 80;
	server_name blog.zls.com;
	root /blog;
	index index.html;
	access_log  /var/log/nginx/blog.zls.com_access_json.log  json;
}

使用Logstash收集nginx日志

[root@elkstack03 conf.d]# cat /etc/logstash/conf.d/nginx_file_es.conf
input{
	file{
		type => "www.zls.com_access"
		path => "/var/log/nginx/www.zls.com_access_json.log"
		start_position => "beginning"
	}
        file{
                type => "blog.zls.com_access"
                path => "/var/log/nginx/blog.zls.com_access_json.log"
                start_position => "beginning"
        }

}

filter{
	json{
		source => "message"
		remove_field => ["message"]
	}
}

output{
	elasticsearch{
		hosts => ["10.0.0.81:9200"]
		index => "%{type}-%{+yyyy.MM.dd}"
		codec => "json"
	}
}


[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash --path.data=/var/lib/logstash/nginx -f /etc/logstash/conf.d/nginx_file_es.conf &