docker-compose部署logstash 8.7

设置文件属组

查看logstash运行用户

# docker run --rm -it docker.elastic.co/logstash/logstash:8.7.0 id

uid=1000(logstash) gid=1000(logstash) groups=1000(logstash)

设置文件属组

# chown 1000.1000 ca.crt

pipelines.yml

# This file is where you define your pipelines. You can define multiple.
# For more information on multiple pipelines, see the documentation:
#   https://www.elastic.co/guide/en/logstash/current/multiple-pipelines.html

- pipeline.id: app1
  path.config: "/usr/share/logstash/pipeline/app1.conf"
- pipeline.id: app2
  path.config: "/usr/share/logstash/pipeline/app2.conf"

app1.conf

input {
  kafka {
    bootstrap_servers => "172.16.3.135:19092,172.16.3.135:29092,172.16.3.135:39092"
    topics => ["app1"]
    codec => "json"
  }
}

output {
  if [fields][type] == "app1" {
    elasticsearch {
      hosts => ["https://172.16.3.9:9200"]
      index => "app1-%{+YYYY.MM.dd}"
      user => "elastic"
      password => "xxxxxxxxxxx"
      cacert => "/usr/share/logstash/config/ca.crt" 
    }
  }

}

app2.conf

input {
  kafka {
    bootstrap_servers => "172.16.3.135:19092,172.16.3.135:29092,172.16.3.135:39092"
    topics => ["app2"]
    codec => "json"
  }
}

output {
  if [fields][type] == "app2" {
    elasticsearch {
      hosts => ["https://172.16.3.9:9200"]
      index => "app2-%{+YYYY.MM.dd}"
      user => "elastic"
      password => "xxxx"
      cacert => "/usr/share/logstash/config/ca.crt"
    }
  }

}

docker-compose.yaml

version: "3.9"

services:
 
  logstash:
    image: docker.elastic.co/logstash/logstash:8.7.0
    container_name: logstash
    restart: always
    volumes:
      - ./settings/pipelines.yml:/usr/share/logstash/config/pipelines.yml
      - ./settings/ca.crt:/usr/share/logstash/config/ca.crt
      - ./pipeline/:/usr/share/logstash/pipeline/
    environment:
      - xpack.monitoring.enabled=false
    networks:
      - es

networks:
  es:
    driver: bridge

参考文档

https://www.elastic.co/guide/en/logstash/current/introduction.html