logstash 收集nginx 日志 linux

linux:

log_format main

'remote_user=$remote_user&ip=$remote_addr&real_ip=$http_x_forwarded_for&log_time=$time_local&request_time=$request_time&host=$http_host&$args&statusbody_bytes_sent=$body_bytes_sent&referer=$http_referer&user_agent=$http_user_agent&forwarded_for=$http_x_forwarded_for'

;

#access_log logs/access.log main;

server {
    listen       443 ssl;
    server_name  t-log.xiyakj.com;
 
    ssl_certificate      /usr/local/nginx/ssl/5879047__xiyakj.com.pem;
    ssl_certificate_key  /usr/local/nginx/ssl/5879047__xiyakj.com.key;
 
    ssl_session_cache    shared:SSL:1m;
    ssl_session_timeout  5m;
    ssl_ciphers  HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers  on;
 
    if ($time_iso8601 ~ "^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})") {
        set $year $1;
        set $month $2;
        set $day $3;
    }
 
    location = /s.gif {
        empty_gif;
    }
 
    access_log /data/logs/nginx/t-log.access_${year}${month}${day}.log main;
}

logstash:

input {
    file {
        # windows 中也使用"/", 而非"\"
        path => "/data/logs/nginx/t-log.access_*.log"
        type => "nginx_access_log"
        start_position => "beginning"
        sincedb_path => "/usr/local/logstash/log_txt/tlog.txt"
        sincedb_write_interval => 15
        stat_interval => "2"
    }
}
 
filter{
   urldecode{
      field => message
   } 
   kv{
      field_split => "&"
   }
}
 
output {
    datahub {
        access_id => "xxxxx"
        access_key => "xxxxxxxxxxxxxxxx"
        endpoint => "http://dh-cn-hangzhou-int-vpc.aliyuncs.com"
        project_name => "ad_log_test"
        topic_name => "ad_test"
        #shard_id => "0"
        #shard_keys => ["thread_id"]
        dirty_data_continue => true
        dirty_data_file => "/usr/local/logstash/log_data/tlog.data"
        dirty_data_file_max_size => 1000
    }
}