ELK集群搭建 --（二）

#收集nginx访问日志

#安装nginx

root@web-1:/usr/local/src# wget https://nginx.org/download/nginx-1.18.0.tar.gz

root@web-1:/usr/local/src# tar xvf nginx-1.18.0.tar.gz

root@web-1:/usr/local/src#cd nginx-1.18.0/

root@web-1:/usr/local/src#./configure --prefix=/apps/nginx

root@web-1:/usr/local/src#make

root@web-1:/usr/local/src#make install

root@web-1:/etc/logstash/conf.d# vim /apps/nginx/conf/nginx.conf

#access_log logs/access.log main;
log_format access_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",' '"url":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"status":"$status"}';
access_log logs/access.log access_json;

 

 #添加配置

root@web-1:/etc/logstash/conf.d# vim /etc/logstash/conf.d/log-to-es.conf

 

 #重启logstash

root@web-1:/etc/logstash/conf.d# systemctl restart logstash.service

#通过 rsyslog 收集 haproxy 日志

[root@haproxy-118 ~]# yum install haproxy

[root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg

listen kibana
bind 10.0.0.118:80
mode http
server kibana1 10.0.0.151:5601 check inter 2s fall 3 rise 5

[root@haproxy-118 ~]# vim /etc/rsyslog.conf

$ModLoad imudp
$UDPServerRun 514

local2.* @@10.0.0.154:2556

[root@haproxy-118 ~]# systemctl restart rsyslog

[root@haproxy-118 ~]# systemctl restart haproxy

root@logstash1:/etc/logstash/conf.d# vim rsyslog.conf

input {
syslog {
host => "10.0.0.154"
port => "2556"
type => "rsyslog"
}
}

output {
if [type] == "rsyslog" {
elasticsearch {
hosts => ["10.0.0.151:9200"]
index => "songyk-rsyslog-%{+YYYY.MM.dd}"
}
}
}
root@logstash1:/etc/logstash/conf.d# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/rsyslog.conf  -t

root@logstash1:/etc/logstash/conf.d# systemctl restart logstash.service 

 

 

#logstash 收集日志并写入 redis 

将web端的日志存入redis，logsstash从redis取出数据，logstash将数据通过haproxy发送给elasticsearch

root@redis:~# apt install redis

root@redis:~# vim /etc/redis/redis.conf

requirepass 12345678

save ""

#save 900 1
#save 300 10
#save 60 10000

 root@logstash1:~# vim /etc/logstash/conf.d/songyk-redis-to-es.conf

input {
redis {
data_type => "list"
key => "nginx-acceslog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"
}

redis {
data_type => "list"
key => "nginx-errorlog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"

}

redis {
data_type => "list"
key => "tomcat-accesslog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"
}

redis {
data_type => "list"
key => "systemlog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"

}

}
output {
if [type] == "nginx-acceslog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-nginx-accesslog-%{+YYY.MM.dd}"
}
}

if [type] == "nginx-errorlog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-nginx-errorlog-%{+YYY.MM.dd}"
}
}

if [type] == "tomcat-acceslog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-tomcat-accesslog-%{+YYY.MM.dd}"
}
}

if [type] == "systemlog" {
elasticsearch {
hosts => ["10.0.0.118:9200"]
index => "songyk-logstash-systemlog-%{+YYY.MM.dd}"
}
}

}

root@web-1:/apps/apache-tomcat-8.5.57# cat /etc/logstash/conf.d/log-to-es.conf
input {
file {
path => "/apps/apache-tomcat-8.5.57/logs/tomcat_access_log.*.log"
type => "tomcat-acceslog"
start_position => "beginning"
stat_interval => "3"
codec => json
}

file {
path => "/var/log/syslog"
type => "systemlog"
start_position => "beginning"
stat_interval => "3"
}

file {
path => "/apps/nginx/logs/access.log"
type => "nginx-acceslog"
start_position => "beginning"
stat_interval => "3"
codec => json
}

file {
path => "/apps/nginx/logs/error.log"
type => "nginx-errorlog"
start_position => "beginning"
stat_interval => "3"
}
}

output {
if [type] == "tomcat-acceslog" {
redis {
data_type => "list"
key => "tomcat-accesslog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"
}
}

if [type] == "systemlog" {
redis {
data_type => "list"
key => "systemlog"
host => "10.0.0.155"
port => "6379"
db => "0"
password => "12345678"

}
}
if [type] == "nginx-acceslog" {
redis {
data_type => "list"
key => "nginx-acceslog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"

}
}

if [type] == "nginx-errorlog" {
redis {
data_type => "list"
key => "nginx-errorlog"
host => "10.0.0.155"
port => "6379"
db => "1"
password => "12345678"
}
}

}

[root@haproxy-118 ~]# vim /etc/haproxy/haproxy.cfg 

listen elasticsearch
bind 10.0.0.118:9200
mode tcp
server es1 10.0.0.151:9200 check inter 2s fall 3 rise 5
server es2 10.0.0.152:9200 check inter 2s fall 3 rise 5
server es3 10.0.0.153:9200 check inter 2s fall 3 rise 5

 #安装metricbeat

 查看服务器指标

root@web-1:/usr/local/src# dpkg -i metricbeat-7.12.1-amd64.deb

root@web-1:/usr/local/src# vim /etc/metricbeat/metricbeat.yml

 

 

 

 未完待续。。。。。。