filebeat+kafka+logstash+es+kibana

filebeat配置

yum 源配置

[root@kafka01 bin]# cat /etc/yum.repos.d/filebeat.repo 
[filebeat-6.x]
name=Elasticsearch repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

 

其他没有用到的参数我都删掉了！

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/messages
output.kafka:
    enabled: true
    hosts: ["192.168.1.7:9092","192.168.1.8:9092","192.168.1.9:9092"]
    topic: messages

 收集多个日志路径和kafka的topic配置

filebeat.inputs:
#messages
- type: log
  enabled: true
  paths:
    - /var/log/messages
  fields:
    log_topics: messages

#secure
- type: log
  enabled: true
  paths:
    - /var/log/secure
  fields:
    log_topics: secure

output.kafka:
    enabled: true
    hosts: ["192.168.1.7:9092","192.168.1.8:9092","192.168.1.9:9092"]
    topic: '%{[fields][log_topics]}'

 logstash配置

"/etc/logstash/conf.d/messages.conf"

input {
        kafka {
                bootstrap_servers => ["192.168.1.7:9092,192.168.1.8:9092,192.168.1.9:9092"]
                group_id => "logstash"
                topics => "messages"
                consumer_threads => 5
        }
}

output {
        elasticsearch {
                hosts => "192.168.1.7:9200"
                index => "messages-%{+YYYY.MM.dd}"
        }

}