filebeat docker 部署

官网：

https://www.elastic.co/guide/en/beats/filebeat/8.10/index.html

https://www.elastic.co/guide/en/beats/filebeat/8.10/running-on-docker.html

 

1. 拉取镜像

sudo docker pull elastic/filebeat:8.10.2

 

2. 准备配置文件

vim /opt/docker/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
   - /opt/logs/*error.log
     # - /opt/docker/log/*.log
  fields:
   #添加新字段可发送至不同topic
   log_topic: sea_test_filebeat_log_topic
  multiline:
        # pattern for error log  多行日志合并，实际项目中一条完整日志可能包含多行信息
        pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
        negate:  true
        match:   after
    #合并最大条数，默认500
     mutiline.max_lines: 1000
      # 这个文件记录日志读取的位置，如果容器重启，可以从记录的位置开始取日志
    #registry_file默认存储在Filebeat的工作目录中，并且命名为".filebeat"。
        #registry_file: /usr/soft/filebeat/data/registry
output.kafka:
  enabled: true
  hosts: ["192.168.18.176:9092","192.168.18.54:9092","192.168.18.199:9092"]
  #根据上面添加字段发送不同topic
  topic: '%{[fields.log_topic]}'
  max_message_bytes: 1000000
  compression: gzip

processors:
 - drop_fields:
     fields: ["host","input","agent","ecs","log","@version","flags"]

logging.level: error
name: sea_app-server-ip

 

测试配置：

filebeat.inputs:
- type: log
  enabled: true
  paths:
   - /var/lib/docker/logger/*/log/*error.log
  fields:
   log_topic: sea_test_filebeat_log2
  multiline:
         # pattern for error log, if start with space or cause by
        pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
        negate:  true
        match:   after
output.kafka:
  enabled: true
  hosts: ["192.168.18.176:9092","192.168.18.54:9092","192.168.18.199:9092"]
  topic: '%{[fields.log_topic]}'
  max_message_bytes: 1000000
  compression: gzip


processors:
 - drop_fields:
     fields: ["host","input","agent","ecs","@version","flags","@metadata"]

# filebeat 的日志级别 建议error , info
logging.level: error
name: sea_app-server-ip

 

 

如果检测多个路径，不同的路径放到不同的topic

filebeat.inputs:
- type: log
  enabled: true
  paths:
   - /opt/docker/logs/*/log/*error.log
  fields:
    log_topic: zhipin_dev_error_log_topic
  multiline:
        pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
        negate:  true
        match:   after
- type: log
  enabled: true
  paths:
   - /opt/docker/logs/*/log/*service-path_record*.log
  fields:
    log_topic: user_req_path_record_topic

output.kafka:
  enabled: true
  hosts: ["192.168.18.176:9092","192.168.18.54:9092","192.168.18.199:9092"]
  topic: '%{[fields.log_topic]}'
  max_message_bytes: 1000000
  compression: gzip


processors:
 - drop_fields:
     fields: ["input","agent","ecs","@version","flags","@metadata"]

logging.level: error
name: sea_app-server-ip192.168.2.1

 

 

 

 

3. 添加对应的权限

 

sudo mkdir -p  /opt/docker/filebeat/data/
sudo chmod -R   666  /opt/docker/filebeat

 

sudo chown 0 filebeat.yml
sudo chmod go-w  filebeat.yml 

 

4.启动服务：

 

sudo docker run -itd \
  --privileged=true \
  --user=root \
  --name=sea_filebeat \
  --restart unless-stopped \
  --network=host \
  -v /var/docker/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:rw \
  -v /var/docker/filebeat/data/:/usr/share/filebeat/data/:rw  \
  -v /var/docker/filebeat/logs/:/usr/share/filebeat/logs:rw  \
 -v /var/lib/docker/logger/:/var/lib/docker/logger/:rw  \
  elastic/filebeat:8.10.2

 -v /var/lib/docker/logger/:/var/lib/docker/logger/:rw  把server 日志挂载到 容器的 /var/lib//docker/logger/ 下,  配置中用该路径

挂载出核心记录数据（日志记录位置，缓存位置信息）注册表中存储每个文件收集的状态(删除后或重新读取数据)

 -v /var/docker/filebeat/data/:/usr/share/filebeat/data

 

配置说明：

多行日志合并

 

#多行合并规则，以时间开头的为一条完整日志，否则合并到上一行（java、python日志都以日期开头）
  multiline.type: pattern
  #中括号日期开头：[2015-08-24 11:49:14,389]
  #multiline.pattern: '^\[[0-9]{4}-[0-9]{2}-[0-9]{2}'
  #日期开头：2015-08-24 11:49:14,389
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  #合并最大条数，默认500
  mutiline.max_lines: 1000
  # 这个文件记录日志读取的位置，如果容器重启，可以从记录的位置开始取日志
  #  registry_file: /usr/soft/filebeat/data/registry