docker-compose搭建elk

一、准备

　　检查自己的docker 和 docker-compose是否安装完毕,切换docker的镜像源

二、安装

　　本次安装的主要组件包括es 、filebeat、kibana、logstash

2.1 先配置组件的挂载点

2.2 配置各组件的相关配置文件

es-->config--->es.yml

cluster.name: "es"
network.host: 0.0.0.0
http.port: 9200
# 开启es跨域
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization,Content-Type
# # 开启安全控制
xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true

　　 filebeat--->config--->filebeat.yml

filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

filebeat.inputs:
- type: log
  paths:
    - /usr/share/filebeat/logs/*.log
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
  multiline.negate: true
  multiline.match: after
  multiline.timeout: 10s

output.logstash:
  hosts: ["logstash:5044"]

kibana--->config--->kibana.yml

server.name: kibana
server.host: "0.0.0.0"
elasticsearch.hosts: [ "http://xxxxx:9200" ] # 修改为自己的ip 
xpack.monitoring.ui.container.elasticsearch.enabled: true
elasticsearch.username: "elastic"  # es账号
elasticsearch.password: "xxxxxx"   # es密码
i18n.locale: zh-CN # 中文

logstash --- > config---> logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.enabled: true
path.config: /usr/share/logstash/config/*.conf
path.logs: /usr/share/logstash/logs

　　logstash --- > config---> logstash-simple.conf

input {
  kafka {
    #id => "kafkaLogs"
    group_id => "logs"
    bootstrap_servers => ["ip:9092"]
    consumer_threads => 5
    topics => ["api-logs"]
    #auto_offset_reset => "latest"
    codec =>json
  }
}
filter {

    date {
       match => ["timestamp","yyyy-MM-dd'T'HH:mm:ss,sss"]
       remove_field => "timestamp"
       target => "@timestamp"
    }

}
output {
  elasticsearch {
    hosts => ["ip:9200"]
    index => "kafka‐%{+YYYY.MM.dd}"
    user => "xxxxx"
    password => "xxxxxx"
  }
}

3、配置docker-compose.yml

version: '3'
 
# 网桥es -> 方便相互通讯
networks:
  es:
 
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - TZ=Asia/Shanghai
      - LANG= en_US.UTF-8
      - ELASTIC_PASSWORD= "xxxxxx" # elastic账号密码 这个地方设置完后，在容器启动后发现密码失败根本登录不进去的情况要进入到容器内部设置，一会细说
    volumes:
      - /docker_vol/elk/es/data:/usr/share/elasticsearch/data
      - /docker_vol/elk/es/config/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml
    hostname: elasticsearch
    restart: always
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - es
 
 
  kibana:
    image: docker.elastic.co/kibana/kibana:7.17.2
    container_name: kibana
    volumes:
      - /docker_vol/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    hostname: kibana
    depends_on:
      - elasticsearch
    links: # 这里用了links 所以 kibana.yml 配置es的时候就不用指定ip
      - elasticsearch
    restart: always
    ports:
      - "5601:5601"
    networks:
      - es
 
  logstash:
    image: docker.elastic.co/logstash/logstash:7.17.2
    container_name: logstash
    hostname: logstash
    restart: always
    environment:
      - TZ=Asia/Shanghai
      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
    volumes:
      - /docker_vol/elk/logstash/config:/usr/share/logstash/config
      - /docker_vol/elk/logstash/log:/usr/share/logstash/logs
    depends_on:
      - elasticsearch
    ports:
      - 9600:9600
      - 5044:5044
   
 
  filebeat:
    image: docker.elastic.co/beats/filebeat:7.17.2
    hostname: filebeat
    restart: always
    depends_on:
      - logstash
    user: root
    volumes:
      - /docker_vol/elk/filebeat/data/filebeat:/usr/share/filebeat/data
      - /docker_vol/elk/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /docker_vol/elk/filebeat/log:/usr/share/filebeat/logs
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - TZ=Asia/Shanghai
    command: ["--strict.perms=false"]