docker-compose搭建elk
一、准备
检查自己的docker 和 docker-compose是否安装完毕,切换docker的镜像源
二、安装
本次安装的主要组件 包括es 、filebeat、kibana、logstash
2.1 先配置组件的挂载点
2.2 配置各组件的相关配置文件
es-->config--->es.yml
cluster.name: "es" network.host: 0.0.0.0 http.port: 9200 # 开启es跨域 http.cors.enabled: true http.cors.allow-origin: "*" http.cors.allow-headers: Authorization,Content-Type # # 开启安全控制 xpack.security.enabled: true xpack.security.transport.ssl.enabled: true
filebeat--->config--->filebeat.yml
filebeat.config: modules: path: ${path.config}/modules.d/*.yml reload.enabled: false processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ filebeat.inputs: - type: log paths: - /usr/share/filebeat/logs/*.log multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}' multiline.negate: true multiline.match: after multiline.timeout: 10s output.logstash: hosts: ["logstash:5044"]
kibana--->config--->kibana.yml
server.name: kibana server.host: "0.0.0.0" elasticsearch.hosts: [ "http://xxxxx:9200" ] # 修改为自己的ip xpack.monitoring.ui.container.elasticsearch.enabled: true elasticsearch.username: "elastic" # es账号 elasticsearch.password: "xxxxxx" # es密码 i18n.locale: zh-CN # 中文
logstash --- > config---> logstash.yml
http.host: "0.0.0.0" xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] xpack.monitoring.enabled: true path.config: /usr/share/logstash/config/*.conf path.logs: /usr/share/logstash/logs
logstash --- > config---> logstash-simple.conf
input {
kafka {
#id => "kafkaLogs"
group_id => "logs"
bootstrap_servers => ["ip:9092"]
consumer_threads => 5
topics => ["api-logs"]
#auto_offset_reset => "latest"
codec =>json
}
}
filter {
date {
match => ["timestamp","yyyy-MM-dd'T'HH:mm:ss,sss"]
remove_field => "timestamp"
target => "@timestamp"
}
}
output {
elasticsearch {
hosts => ["ip:9200"]
index => "kafka‐%{+YYYY.MM.dd}"
user => "xxxxx"
password => "xxxxxx"
}
}
3、配置docker-compose.yml
version: '3' # 网桥es -> 方便相互通讯 networks: es: services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.2 container_name: elasticsearch environment: - discovery.type=single-node - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - TZ=Asia/Shanghai - LANG= en_US.UTF-8 - ELASTIC_PASSWORD= "xxxxxx" # elastic账号密码 这个地方设置完后,在容器启动后发现密码失败根本登录不进去的情况要进入到容器内部设置,一会细说 volumes: - /docker_vol/elk/es/data:/usr/share/elasticsearch/data - /docker_vol/elk/es/config/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml hostname: elasticsearch restart: always ports: - 9200:9200 - 9300:9300 networks: - es kibana: image: docker.elastic.co/kibana/kibana:7.17.2 container_name: kibana volumes: - /docker_vol/elk/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml hostname: kibana depends_on: - elasticsearch links: # 这里用了links 所以 kibana.yml 配置es的时候就不用指定ip - elasticsearch restart: always ports: - "5601:5601" networks: - es logstash: image: docker.elastic.co/logstash/logstash:7.17.2 container_name: logstash hostname: logstash restart: always environment: - TZ=Asia/Shanghai - "ES_JAVA_OPTS=-Xms1g -Xmx1g" volumes: - /docker_vol/elk/logstash/config:/usr/share/logstash/config - /docker_vol/elk/logstash/log:/usr/share/logstash/logs depends_on: - elasticsearch ports: - 9600:9600 - 5044:5044 filebeat: image: docker.elastic.co/beats/filebeat:7.17.2 hostname: filebeat restart: always depends_on: - logstash user: root volumes: - /docker_vol/elk/filebeat/data/filebeat:/usr/share/filebeat/data - /docker_vol/elk/filebeat/config/filebeat.yml:/usr/share/filebeat/filebeat.yml - /docker_vol/elk/filebeat/log:/usr/share/filebeat/logs - /var/run/docker.sock:/var/run/docker.sock environment: - TZ=Asia/Shanghai command: ["--strict.perms=false"]
三、启动
cd 到 docker-compose.yml 所在层级 docker-compose up -d