ELK-第二集

收集nginx日志和系统日志写到kafaka在用logstash读取出来写到elasticsearch
##node1 把nginx日志写到kafka里面
[root@node1 conf.d]# vim /etc/logstash/conf.d/nginx.conf
 
input{
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx-access-log-1105"
    start_position => "beginning"
    stat_interval => "2"
    codec => "json"
  }
  file {
    path => "/var/log/messages"
    type => "system-log-1105"
    start_position => "beginning"
    stat_interval => "2"
  }
}
 
 
output {
  if [type] == "nginx-access-log-1105" {
  kafka {
    bootstrap_servers => "192.168.1.106:9092"
    topic_id => "nginx-accesslog-1105"
    codec => "json"
}
}
  if [type] == "system-log-1105" {
    kafka {
    bootstrap_servers => "192.168.1.106:9092"
    topic_id => "system-log-1105"
    codec => "json"
}}
}
 
 
node2 ##从kafaka读出来写到elasticsearch
 
input {
  kafka {
    bootstrap_servers => "192.168.1.105:9092"
    topics => "nginx-accesslog-1105"
    group_id => "nginx-access-log"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
  }
    kafka {
    bootstrap_servers => "192.168.1.105:9092"
    topics => "system-log-1105"
    group_id => "nginx-access-log"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
  }
}
 
output {
#  stdout {
#    codec => "rubydebug"
#  }
  if [type] == "nginx-access-log-1105" {
  elasticsearch {
  hosts => ["192.168.1.105:9200"]
  index => "logstash-nginx-access-log-1105-%{+YYYY.MM.dd}"
  }}
  if [type] == "system-log-1105" {
    elasticsearch {
  hosts => ["192.168.1.106:9200"]
  index => "logstash-systemzzz-log-1105-%{+YYYY.MM.dd}"
}
}
}

在添加到kibana里面

##使用fileeat来收集日志写入kafka

node1 上传filebeat-5.6.5-x86_64.rpm

yum install filebeat-5.6.5-x86_64.rpm -y

systemctl stop logstash.service

[root@node1 tmp]# grep -v "#" /etc/filebeat/filebeat.yml | grep -v "^$"
filebeat.prospectors:
- input_type: log
paths:
- /var/log/*.log
- /var/log/messages
exclude_lines: ["^DBG"]
exclude_files: [".gz$"]
document_type: "system-log-1105-filebeat"
output.file:
path: "/tmp"
filename: "filebeat.txt"

output.logstash:
hosts: ["192.168.1.105:5044"] #logstash 服务器地址，可以是多个
enabled: true #是否开启输出至logstash,默认即为true
worker: 1 #工作线程数
comperession_level: 3 #压缩级别
#loadbalance: true #多个输出的时候开启负载

[root@node1 src]# vim /etc/logstash/conf.d/filebate.conf

input {
beats {
    port => "5044"
    codec => "json"
}
}
output {
if [type] == "system-log-1105-filebeat" {
kafka {
    bootstrap_servers => "192.168.1.105:9092"
    topic_id => "system-log-filebeat-1105"
    codec => "json"
}
}
}

[root@node1 conf.d]# systemctl restart logstash.service

[root@node1 conf.d]# /usr/local/kafka/bin/kafka-topics.sh --list --zookeeper 192.168.1.105:2181,192.168.1.106:2181,192.168.1.101:2181

node2##

[root@node2 conf.d]# vim kafka-es.conf

input {
kafka {
    bootstrap_servers => "192.168.1.105:9092"
    topics => "system-log-filebeat-1105"
    group_id => "system-log-filebeat"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
}
}

output {
# stdout {
# codec => "rubydebug"
# }
if [type] == "system-log-1105-filebeat" {
elasticsearch {
hosts => ["192.168.1.106:9200"]
index => "system-log-1105-filebeat-%{+YYYY.MM.dd}"
}
}
}

[root@node2 conf.d]# systemctl restart logstash.service

#测试在node1 /var/log/messages里添加东西，然后去9100端口去检查，如果有证明正常，然后添加到kibana里面。

##流程是，filebeat配置文件从messages里面读取然后输出到192.168.1.105:5044端口上，然后logstash从本地的5044端口上读取写入kafka:9092端口，然后node2的logstash从input 192.168.1.105的9092端口读取内容output输出到elasticsearch上。

###收集nginx

[root@node1 conf.d]# vim /etc/filebeat/filebeat.yml

- input_type: log
paths:
- /var/log/nginx/access.log
exclude_lines: ["^DBG"]
exclude_files: [".gz$"]
document_type: "nginx-accesslog-1105-filebeat"

[root@node1 src]# vim /etc/logstash/conf.d/filebate.conf

output {
if [type] == "nginx-accesslog-1105-filebeat" {
kafka {
    bootstrap_servers => "192.168.1.105:9092"
    topic_id => "nginx-accesslog-filebeat-1105"
    codec => "json"
}}
}

node2

input {
kafka {
bootstrap_servers => "192.168.1.105:9092"
topics => "nginx-accesslog-filebeat-1105"
group_id => "nginx-accesslog-filebeat"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
}
}

output {
if [type] == "nginx-accesslog-1105-filebeat" {
elasticsearch {
hosts => ["192.168.1.106:9200"]
index => "logstash-nginx-accesslog-1105-filebeat-%{+YYYY.MM.dd}"
}
}
}

[root@node2 conf.d]# systemctl restart logstash.service

##收集java日志

[root@node1 conf.d]# vim /etc/logstash/conf.d/java.conf

input {
file {
    path => "/var/log/logstash/logstash-plain.log"
    type => "javalog"
    codec => multiline {
    pattern => "^\[(\d{4}-\d{2}-\d{2})"
    negate => true
    what => "previous"
   }}
}

output {
elasticsearch {
hosts => ["192.168.1.105:9200"]
index => "javalog-1105-%{+YYYY.MM}"
}
}