filebeat+redis+logstash+elasticsearch基本配置--适用于6.4版本

filebeat配置：

filebeat.inputs:
- type: log
enabled: true
paths:
- /opt/xxxx.log
fields:                                      在filebeat收集的消息后面新增加字段，用于后面logstash的区分，分别放置于不同的索引
  service: xxx               
  fields_under_root: true

- type: log
enabled: true
paths:
- /xxx.log
fields:
service: xxxxx
fields_under_root: true
multiline.pattern: '^Caused by'      ---用于收集java异常日志
multiline.negate: true
multiline.match: after
multiline.max_lines: 1000
output.redis:
hosts: ["xxx:6379"]
db: 0
time: 10
key: "xx"            放置于redis中的key值

  

logstash配置：

input {
redis {
host => "127.0.0.1"
data_type => "list"
key => "xxx"        与上述配置filebeat中的redis值保持一致
}

}

output {

if [service] == "xxx" { 
elasticsearch {
hosts => ["http://xxx:9200"]
index => "xxx-%{+YYYY.MM.dd}"
}
}

if [service] == "xxx" {
elasticsearch {
hosts => ["http://10.157.25.7:9200"]
index => "xxx-%{+YYYY.MM.dd}"
}
}

}

 

 

elasticsearch配置:

cluster.name: my-cluster
node.name: es-node-3
path.data: /data/es/
path.logs: /data/logs/
network.host:xxxx
http.port: 9200
transport.tcp.port: 9300
discovery.zen.ping.unicast.hosts: ["xx","xx","xx"]
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping_timeout: 60s