ubuntu22.04 部署 filebeat 8.7

下载filebeat

# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-8.7.0-linux-x86_64.tar.gz

创建数据目录

# mkdir -pv /data/apps/filbeat/{data,logs}

安装filebeat

# tar xzvf filebeat-8.7.0-linux-x86_64.tar.gz -C /usr/local/
# ln -sv /usr/local/filebeat-8.7.0-linux-x86_64 /usr/local/filebeat

filebeat.service

[Unit]
Description=Filebeat sends log files to Logstash or directly to Elasticsearch.
Documentation=https://www.elastic.co/beats/filebeat
Wants=network-online.target
After=network-online.target

[Service]

UMask=0027
Environment="GODEBUG='madvdontneed=1'"
Environment="BEAT_LOG_OPTS="
Environment="BEAT_CONFIG_OPTS=-c /usr/local/filebeat/filebeat.yml"
Environment="BEAT_PATH_OPTS=--path.home /usr/local/filebeat --path.config /usr/local/filebeat --path.data /data/apps/filebeat/data --path.logs /data/apps/filebeat/logs"
ExecStart=/usr/local/filebeat/filebeat --environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
Restart=always

[Install]
WantedBy=multi-user.target

CA 证书

证书信息

# openssl x509 -fingerprint -sha256  -in certs/ca/ca.crt 
sha256 Fingerprint=76:76:94:E1:79:5E:F0:70:44:E8:EA:7F:CC:61:9E:F6:7A:87:D9:C1:FD:A2:57:53:92:ED:0E:B3:7F:59:8A:07
-----BEGIN CERTIFICATE-----
MIIDSTCCAjGgAwIBAgIUB4YfjOt/EhK11Xxk8XU/ZcbmzCQwDQYJKoZIhvcNAQEL
BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
cmF0ZWQgQ0EwHhcNMjMwNDExMDk1NDA5WhcNMjYwNDEwMDk1NDA5WjA0MTIwMAYD
VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsM/76re0u8JMyAyBaW+jRa
XgfS4iZ+Pt5c4sMHJmO9wBud8madghVEQOzCwvT5b23ah7HUBRxv/LUo/XehPcQb
fv/ACA5VnsXo6JBQWBVv+YJGwQTQifSVO/APeI30XMmxS49aik5YhVZOfytjSl3d
y2MHhxNis3O7FaJLnDyHwcU7dQOeXtPX7s1PXeJPYl5xbE+LoVR92osC2J4nDb95
iCWwRHToOXEHOaDg3mRQ3W4xVSVrvfZEFUCgdGI1Hsd7c2mbGGq6AAm5o23eGvUM
ew9YSeEDR+rOg1xGpYf9mdl6lj/j6ciom2N5LRIy9E4P4T0W0XSqz6EcCudBqb0C
AwEAAaNTMFEwHQYDVR0OBBYEFM3MIU8ioLunGIuHJuC1TgkTqXq9MB8GA1UdIwQY
MBaAFM3MIU8ioLunGIuHJuC1TgkTqXq9MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
hvcNAQELBQADggEBACej4e+57p4MHljKp0r1IGd+vGaitoIv3/jKlzxNWlu609cJ
tqzqumKtbi7m5lVuRmM3DY9h5/SDlotcTSbivdtoBVdqQ6X3urFn6Ha7olle/Nto
QLscX7HdOzqPdBIopG8BwEfTbotJMGSoo8AbDy7B0UHZqR6q4A24zuJwiF0MiwHA
1waVM9kaJle5ds0E0Ybo8pxe3wac7WC1D4PrQcWfN3eSY3aE7AcUDV8KPajftL1q
SveOnW1tvF+DJdfmtUvYAuq6xjW41jey8/72AaYkzRRU6gA64bWxtqgcmbiFL4ox
aWawju0unBK76NXrrWCLICg1FcaTKWqYhC2fSFc=
-----END CERTIFICATE-----

fingerprint

# openssl x509 -fingerprint -sha256 -noout -in certs/ca/ca.crt | awk --field-separator="=" '{print $2}' | sed 's/://g'
767694E1795EF07044E8EA7FCC619EF67A87D9C1FDA2575392ED0EB37F598A07

filebeat.yml

filebeat.inputs:
- type: filestream 
  enabled: true
  id: my-filestream-id
  paths:
    - /var/log/system.log
    - /var/log/wifi.log
  exclude_lines: ['filebeat','Filebeat']
  fields:
    source: syslog
- type: filestream 
  enabled: false
  id: nginx-filestream-id
  paths:
    - "/var/log/nginx/*"
  fields:
    source: nginx
  parsers:
    - ndjson:
        target: ""
        overwrite_keys: true

- type: filestream 
  enabled: false
  id: tomcat-filestream-id
  paths:
    - "/var/log/tomcat/*"
  fields:
    source: tomcat
  parsers:
    - multiline:
        type: pattern
        pattern: '^[[:space:]]+(at|\.{3})[[:space:]]+\b|^Caused by:'
        negate: false
        match: after

filebeat.config.modules:
  # Glob pattern for configuration loading
  path: ${path.config}/modules.d/*.yml

  # Set to true to enable config reloading
  reload.enabled: true

  # Period on which files under path should be checked for changes
  #reload.period: 10s

output.elasticsearch:
  hosts: ["myEShost:9200"]
  protocol: "https"
  username: "filebeat_internal"
  password: "YOUR_PASSWORD" 
  ssl:
    enabled: true
    indices:
    - index: "syslog-%{+yyyy.MM.dd}"
      when.equals:
        fields:
          source: syslog
    - index: "nginx-%{+yyyy.MM.dd}"
      when.equals:
        fields:
          source: nginx
    certificate_authorities:
      - |
        -----BEGIN CERTIFICATE-----
        MIIDSTCCAjGgAwIBAgIUB4YfjOt/EhK11Xxk8XU/ZcbmzCQwDQYJKoZIhvcNAQEL
        BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l
        cmF0ZWQgQ0EwHhcNMjMwNDExMDk1NDA5WhcNMjYwNDEwMDk1NDA5WjA0MTIwMAYD
        VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC
        ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJsM/76re0u8JMyAyBaW+jRa
        XgfS4iZ+Pt5c4sMHJmO9wBud8madghVEQOzCwvT5b23ah7HUBRxv/LUo/XehPcQb
        fv/ACA5VnsXo6JBQWBVv+YJGwQTQifSVO/APeI30XMmxS49aik5YhVZOfytjSl3d
        y2MHhxNis3O7FaJLnDyHwcU7dQOeXtPX7s1PXeJPYl5xbE+LoVR92osC2J4nDb95
        iCWwRHToOXEHOaDg3mRQ3W4xVSVrvfZEFUCgdGI1Hsd7c2mbGGq6AAm5o23eGvUM
        ew9YSeEDR+rOg1xGpYf9mdl6lj/j6ciom2N5LRIy9E4P4T0W0XSqz6EcCudBqb0C
        AwEAAaNTMFEwHQYDVR0OBBYEFM3MIU8ioLunGIuHJuC1TgkTqXq9MB8GA1UdIwQY
        MBaAFM3MIU8ioLunGIuHJuC1TgkTqXq9MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
        hvcNAQELBQADggEBACej4e+57p4MHljKp0r1IGd+vGaitoIv3/jKlzxNWlu609cJ
        tqzqumKtbi7m5lVuRmM3DY9h5/SDlotcTSbivdtoBVdqQ6X3urFn6Ha7olle/Nto
        QLscX7HdOzqPdBIopG8BwEfTbotJMGSoo8AbDy7B0UHZqR6q4A24zuJwiF0MiwHA
        1waVM9kaJle5ds0E0Ybo8pxe3wac7WC1D4PrQcWfN3eSY3aE7AcUDV8KPajftL1q
        SveOnW1tvF+DJdfmtUvYAuq6xjW41jey8/72AaYkzRRU6gA64bWxtqgcmbiFL4ox
        aWawju0unBK76NXrrWCLICg1FcaTKWqYhC2fSFc=
        -----END CERTIFICATE-----
    ca_trusted_fingerprint: "767694E1795EF07044E8EA7FCC619EF67A87D9C1FDA2575392ED0EB37F598A07"

setup.kibana:
  host: "mykibanahost:5601" 
  username: "my_kibana_user"  
  password: "{pwd}"

收集数据模块

查看可用的模块

# filebeat modules list

启用模块

# /usr/local/filebeat/filebeat modules enable nginx
Enabled nginx

修改nginx模块

# cat modules.d/nginx.yml

# Module: nginx
# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-nginx.html

- module: nginx
  # Access logs
  access:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/nginx/access.log*"]

  # Error logs
  error:
    enabled: true

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    var.paths: ["/var/log/nginx/access.log*"]

  # Ingress-nginx controller logs. This is disabled by default. It could be used in Kubernetes environments to parse ingress-nginx logs
  ingress_controller:
    enabled: false

    # Set custom paths for the log files. If left empty,
    # Filebeat will choose the paths depending on your OS.
    #var.paths:

查看索引

输出到kafka

output.kafka:
  # initial brokers for reading cluster metadata
  hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"]

  # message topic selection + partitioning
  topic: '%{[fields.log_topic]}'   # web-app
  partition.round_robin:
    reachable_only: false

  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

参考文档

https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-installation-configuration.html