Filebeat配置

# 1.安装
yum install filebeat
# 或者
tar -zxvf filebeat-8.12.0-linux-x86_64.tar.gz -C /etc/filebeat
mv filebeat-8.12.0-linux-x86_64 filebeat

# 2.修改配置
vim /etc/systemd/system/filebeat.service
vim /etc/filebeat/filebeat/filebeat.yml

# 3.启动
systemctl enable filebeat
systemctl start filebeat
systemctl list-unit-files |grep enabled

filebeat.service

[Unit]
Description=Filebeat sends log files to Kafka
Documentation=https://www.elastic.co/products/beats/filebeat
Wants=network-online.target
After=network-online.target

[Service]

Environment="BEAT_CONFIG_OPTS=-c /etc/filebeat/filebeat/filebeat.yml"
Environment="BEAT_PATH_OPTS=-path.home /etc/filebeat/filebeat/  -path.config /etc/filebeat/filebeat/ -path.data /etc/filebeat/filebeat/data -path.logs /var/log/filebeat"
ExecStart=/etc/filebeat/filebeat/filebeat  $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
Restart=always

[Install]
WantedBy=multi-user.target

filebeat.yml

logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat
  keepfiles: 7
  permissions: 0644
logging:
  json: true
logging.metrics:
  enabled: true
  period: 60

  
- type: filestream
  id: sase-pe-nginx
  enabled: true
  paths:
    - /data/waf/logs/access_log/*/access.log
  fields:
    log_topic: access
  prospector.scanner.check_interval: 1
  prospector.scanner.exclude_files: ['\.gz$']


processors:
- drop_fields:
    fields: ["host","log", "input", "agent","ecs"]



output.kafka:
  enabled: true
  hosts: ["hadoop102:9092"]
  topic: '%{[fields.log_topic]}'
  sasl.mechanism: "PLAIN"
  username: "admin"
  password: "admin"
  codec.format:
    string: '%{[message]}'