docker-compose多服务器部署ELK

多服务器构建ELK

• es使用1主2从,logstash转发,kibana展现，本文ELK版本使用7.16.1
• 如果单机测试，请查看docker-compose单服务器部署ELK

• es-master:10.10.210.96
• es-slave1:10.10.210.97
• es-slave2:10.10.210.98

es-master

目录结构

- src
  - data
  - logs
  - docker-compose.yml
  - elasticsearch.yml

docker-compose.yml

version: "3"
services:
  es-master: 
    image: elasticsearch:7.16.1
    container_name: es-master
    environment:
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
    ports:
      - "9200:9200"
      - "9300:9300"
    expose: 
      - "9200"                                    
      - "9300"    
    restart: always
    volumes:       
      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./data:/usr/share/elasticsearch/data:rw
      - ./logs/:/usr/share/elasticsearch/logs

elasticsearch.yml

cluster.name: es-cluster
node.name: es-master
node.roles: ["master"]
network.host: 0.0.0.0
network.publish_host: 10.10.210.96
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.10.210.96:9300"]
cluster.initial_master_nodes: ["es-master"]

es-slave1

目录结构

- src
  - data
  - logs
  - docker-compose.yml
  - elasticsearch.yml

docker-compose.yml

version: "3"
services:
  es-slave1: 
    image: elasticsearch:7.16.1
    container_name: es-slave1
    environment: 
      - ES_JAVA_OPTS=-Xms512m -Xmx512m  
    ports:
      - "9200:9200"
      - "9300:9300"
    expose: 
      - "9200"                                    
      - "9300"    
    restart: always
    volumes:     
      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./data:/usr/share/elasticsearch/data:rw
      - ./logs/:/usr/share/elasticsearch/logs

elasticsearch.yml

cluster.name: es-cluster
node.name: es-slave1
node.roles: ["data"]
network.host: 0.0.0.0
network.publish_host: 10.10.210.97
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.10.210.96:9300"]
cluster.initial_master_nodes: ["es-master"]

es-slave2

目录结构

- src
  - data
  - logs
  - docker-compose.yml
  - elasticsearch.yml

docker-compose.yml

version: "3"
services:
  es-slave2: 
    image: elasticsearch:7.16.1
    container_name: es-slave2
    environment:
      - ES_JAVA_OPTS=-Xms512m -Xmx512m  
    ports:
      - "9200:9200"
      - "9300:9300"
    expose: 
      - "9200"                                    
      - "9300"    
    restart: always
    volumes:    
      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./data:/usr/share/elasticsearch/data:rw
      - ./logs:/usr/share/elasticsearch/logs:rw

elasticsearch.yml

cluster.name: es-cluster
node.name: es-slave2
node.roles: ["data"]
network.host: 0.0.0.0
network.publish_host: 10.10.210.98
http.cors.enabled: true
http.cors.allow-origin: "*"
http.port: 9200
transport.port: 9300
discovery.seed_hosts: ["10.10.210.96:9300"]
cluster.initial_master_nodes: ["es-master"]

logstash

• 使用grok解析数据

目录结构

- src
  - config
    - logstash.conf
    - logstash.yml
  - docker-compose.yml

docker-compose.yml

version: "3"
services:
  logstash:
    image: logstash:7.16.1
    container_name: logstash
    ports:
      - "9600:9600"
      - "5044:5044"
    restart: always
    environment:
      - XPACK_MONITORING_ENABLED=false
    volumes:
      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:rw
      - ./config/logstash.yml:/usr/share/logstash/config/logstash.yml:rw

logstash.conf

input{
    beats {
        port => 5044
    }
}
filter {
    ruby {
        code => "event.timestamp.time.localtime"
    }
}
output {
    stdout {
        codec => rubydebug
    }
}

logstash.yml

http.host: 0.0.0.0
xpack.monitoring.enabled: false

kibana

目录结构

- src
  - docker-compose.yml

docker-compose.yml

version: "3"
services:
    kibana:
        image: kibana:7.16.1
        container_name: kibana
        environment:
            - ELASTICSEARCH_HOSTS=["http://10.10.210.96:9200"]
            - I18N_LOCALE=zh-CN
        ports:
            - "5601:5601"