ELK环境搭建

一、结构图

组件的单节点安装步骤见: https://blog.csdn.net/lwxvgdv/article/details/140122444

elk配置见: https://blog.csdn.net/weixin_50236289/article/details/140232092

二、各组件安装及配置

1.filebeat

wget https://artifacts.elastic.co/downloads/filebeat/filebeat-7.17.2-x86_64.rpm
yum -y install filebeat-7.17.2-x86_64.rpm

# 修改filebeat配置文件
[root@web filebeat]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access_json.log
 json.keys_under_root: true #默认false,只识别为普通文本,会将全部日志数据存储至message字段，改为true则会以Json格式存储,json必选
 json.overwrite_keys: true  #设为true,使用json格式日志中自定义的key替代默认的message字段,此项可选
 tags: ["nginx-access"]     #指定tag,用于分类
  
- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
 tags: ["nginx-error"]
  
output.elasticsearch:
  hosts: ["11.0.1.131:9200","11.0.1.136:9200","11.0.1.137:9200"]        
  indices:
  - index: "nginx-access-%{[agent.version]}-%{+yyy.MM.dd}"
    when.contains:
      tags: "nginx-access"   #如果记志中有access的tag,就记录到nginx-access的索引中
  - index: "nginx-error-%{[agent.version]}-%{+yyy.MM.dd}"
    when.contains:
      tags: "nginx-error"   #如果记志中有error的tag,就记录到nginx-error的索引中
  

2.kafka

https://mp.weixin.qq.com/s?__biz=Mzg4ODQ1NTE2Mg==&mid=2247567260&idx=1&sn=35d978fb08afb9cd34fdd68f9649954e&chksm=ce333b0e4ca6ef30c5454ff96bddc0b3f160156fc11d9ef79494bbe6ef7f258848e098a4712c&scene=27

创建topic

kafka-topics.sh --create --topic filebeat-log --bootstrap-server localhost:9092

 

3.logstash

wget https://artifacts.elastic.co/downloads/logstash/logstash-7.17.2-x86_64.rpm
yum -y install logstash-7.17.2-x86_64.rpm

配置

[root@k8s-master conf.d]# cat kafka-to-es.conf 
input {
 kafka {
   bootstrap_servers => "192.168.0.120:9092"
   topics => "filebeat-log"
   group_id => "logstash" #消费者组的名称
   codec => "json"
   consumer_threads => "8" #建议设置为和kafka的分区相同的值为线程数
 }
}
output {
  if "nginx-log" in [tags] {
   elasticsearch {
     hosts => ["192.168.0.121:9201"]
     #hosts => ["192.168.0.120:9201","192.168.0.121:9201"]
     index => "logstash-kafka-nginx-log-%{+YYYY.MM.dd}"
   }
 }
}
[root@k8s-master conf.d]# pwd
/etc/logstash/conf.d

systemctl enable logstash
systemctl start logstash

4.elasticsearch