ELK之收集Nginx、Tomcat的json格式日志

1.安装Nginx

yum -y install nginx
vim /etc/nginx/nginx.conf
# 修改日志格式为json格式,并创建一个nginxweb的网站目录
log_format access_json '{"@timestamp":"$time_iso8601",'
                           '"host":"$server_addr",'
                           '"clientip":"$remote_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"upstreamtime":"$upstream_response_time",'
                           '"upstreamhost":"$upstream_addr",'
                           '"http_host":"$host",'
                           '"url":"$uri",'
                           '"domain":"$host",'
                           '"xff":"$http_x_forwarded_for",'
                           '"referer":"$http_referer",'
                           '"status":"$status"}';
access_log  /var/log/nginx/access.log  access_json;
vim /etc/nginx/conf.d/nginxweb.conf
server {
    listen       80;
    server_name  10.0.0.22;

    location /nginxweb {
        root html;
        index index.html index.htm;
    }
    error_page  404              /404.html;

    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }
}

mkdir /usr/share/nginx/html/nginxweb
echo "<h1> welcome to use Nginx" </h1> /usr/share/nginx/html/nginxweb/index.html
nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
systemctl start nginx
# 访问http://10.0.0.22/nginxweb/时一直报404,查了一下,发现/etc/nginx/下没有静态文件
ln -s /usr/share/nginx/html/ /etc/nginx/

2.配置logstash

vim /etc/logstash/conf.d/nginx-accesslog.conf 
input{
    file {
        path => "/var/log/nginx/access.log"
        type => "nginx-access-log"
        start_position => "beginning"
        stat_interval => "2"
    }
}

output{
   elasticsearch {
        hosts => ["10.0.0.22:9200"]
        index => "logstash-nginx-access-log-%{+YYYY.MM.dd}"
   }
}

systemctl restart logstash
# 压力测试
yum -y install httpd-tools
ab -n 800 -c 100  http://10.0.0.22/nginxweb/index.html
-n:requests Number of requests to perform 要执行的请求数
-c:Concurrency 并发

nginx属于 adm 组,使用 logstash 读取日志,可能产生权限异常
usermod -G adm logstash

  在elasticsearch-head页面查看日志时,点击A-index,再点击B-index时,会把A-index的所有内容与B-index相合并,再点一下A-index,就只剩B-index的内容了.

3.安装tomcat

wget http://mirror.bit.edu.cn/apache/tomcat/tomcat-8/v8.5.37/bin/apache-tomcat-8.5.37.tar.gz
tar xf apache-tomcat-8.5.37.tar.gz
ln -s /usr/local/src/apache-tomcat-8.5.37 /usr/local/src/apache-tomcat
cd /usr/local/src/apache-tomcat/webapps/
mkdir webpage
echo "this is tomcat web page" > webpage/index.html
../bin/catalina.sh start
# 访问http://10.0.0.22:8080/webpage/index.html
cd ..
tail logs/localhost_access_log.2019-02-06.txt 
10.0.0.1 - - [06/Feb/2019:01:34:30 +0800] "GET /webpage/index.html HTTP/1.1" 200 24
10.0.0.1 - - [06/Feb/2019:01:34:31 +0800] "GET /favicon.ico HTTP/1.1" 200 21630
cd conf/
cp server.xml{,.bak}
vim server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
prefix="tomcat_access_log" suffix=".log"
pattern="{"clientip":"%h","ClientUser":"%l",
"authenticated":"%u","AccessTime":"%t",
"method":"%r","status":"%s",
"SendBytes":"%b","Query?string":"%q",
"partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>

cd ..
rm -rf logs/*
./bin/catalina.sh stop
./bin/catalina.sh start
tail logs/tomcat_access_log.2019-02-06.log

vim /etc/logstash/conf.d/tomcat_accesslog.conf 
input {
  file {
    path => "/usr/local/src/apache-tomcat/logs/tomcat_access_log.*.log"
    type => "tomcat-access"
    start_position => "beginning"
    stat_interval => "2"
    }
}

output {
  if [type] == "tomcat-access" {
  elasticsearch {
    hosts => ["10.0.0.22:9200"]
    index => "logstash-tomcat1022-access-%{+YYYY.MM.dd}"
    }
  }
}
systemctl restart logstash
# 无法出现tomcat的数据索引,权限有问题
cd /usr/local/src/apache-tomcat/
chmod 755 logs/
chmod 666 tomcat_access_log.2019-02-06.log

权限改成644都不行

 

Nginx的json格式日志收集:http://blog.51cto.com/jinlong/2055173

Tomcat的json格式日志收集:http://blog.51cto.com/jinlong/2055379