filebeat+elasticsearch+kibana

一、到elasticsearch官网下载 filebeat+elasticsearch+kibana

http://www.elasticsearch.cn/

 

二、安装filebeat

tar -xzvf filebeat-7.16.3-linux-x86_64.tar.gz -C /opt
cd /opt
mv filebeat-7.16.3-linux-x86_64 filebeat
groupadd -g 1004 fbeat
useradd -u 1004 fbeat -g 1004 -s /sbin/nologin -M
chown -R fbeat.fbeat filebeat

 

systemd纳管filebeat

cat <<efo> /usr/lib/systemd/system/filebeat.service
[Unit]
Description=filebeat 7.16.3
After=syslog.target network.target docker.service

[Service]
#Type=simple
User=fbeat
Group=fbeat
ExecStart=/opt/filebeat/filebeat -c /opt/filebeat/filebeat.yml
PrivateTmp=true

[Install]
WantedBy=multi-user.target

efo

systemctl daemon-reload

systemctl enable filebeat --now

filebeat.yml模板

filebeat.inputs:
- type: log
  paths:
    - /mnt/logs/*.json
#日志多行匹配根据需求删除注释
#  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
#  multiline.negate: true
#  multiline.match: after
#如果是json格式日志需要添加以下三行
  json.key_under_root: true
  json.overwrite_keys: true
  json.message_key: log
 #encoding: gbk   日志编码
  fields_under_root: true
  fields:
    source: system-index-day
    app: elk
    eventType: trigger
    metric: ELK日志监控告警
    resourceIp: 本机ip
    resourceName: "system-index-day"
  clean_*: 48h
  close_*: 5m
  idle_timeout: 30s
  scan_frequency: 8s
  processors:
    - drop_fields:
        fields: ["host", "tags", "ecs", "prospector", "agent", "input", "beat", "offset"]
        ignore_missing: true
 
- type: log
  paths:
    - /var/log/dmesg
    - /var/log/messages
    - /var/log/boot.log
    - /var/log/audit/audit.log
    - /var/log/secure
    - /var/log/cron
#    Ubuntu
    - /var/log/kern.log
    - /var/log/auth.log
    - /var/log/syslog
    - /var/log/bootstrap.log
  fields_under_root: true
  fields:
    source: system-index-day
    ip: 本机ip
  clean_*: 48h
  close_*: 5m
  idle_timeout: 30s
  scan_frequency: 8s
  processors:
    - drop_fields:
        fields: ["host", "tags", "ecs", "prospector", "agent", "input", "beat", "offset"]
        ignore_missing: true
 
setup.template.settings:
   index.number_of_shards: 1
   index.number_of_replicas: 1
 
setup.ilm.enabled: false
setup.template.enabled: true
setup.template.overwrite: true
setup.kibana:
  host: "http://xxxx.domain.com:5601"
#对接ES
output.elasticsearch:
  host: ["xxxx.domain.com:9200"]
#对接kafka
output.kafka:
  hosts: ["xxxx.domain.com:9092"]
  topic: 'xxxx'
  partition.round_robin:
    reachable_only: false
  required_acks: 1
  compression: gzip
  max_message_bytes: 10000000
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - add_docker_metadata: ~

 

三、安装es

vi /etc/sysctl.conf
行尾增加:  vm.max_map_count=655360
sysctl -p

groupadd elasticsearch
useradd -m -g elasticsearch elasticsearch
chown -R elasticsearch /elasticsearch
chmod 755 -R /elasticsearch/config/
chmod 777 -R /elasticsearch/logs/
chmod 777 -R /elasticsearch/data/

配置 elasticsearch.yml

vi elasticsearch.yml
cluster.name: elk-test
node.name: elk-test-node1
network.host: 0.0.0.0
network.publish_host: 节点IP
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization,X-Requested-With,Content-Length,Content-Type
discovery.seed_hosts: ["多个节点IP以逗号分隔"] #集群成员
cluster.initial_master_nodes: ["主节点IP"] #指定主节点
node.master: true
node.data: true

#启动es
docker run -d --name elasticsearch --restart=always  -p 9200:9200 -p 9300:9300 -v /elasticsearch/config:/usr/share/elasticsearch/config -v /elasticsearch/data:/usr/share/elasticsearch/data -v /elasticsearch/logs:/usr/share/elasticsearch/logs  elasticsearch:7.16.3

 

四、安装kibana

配置 kibana.yml

server.publicBaseUrl: "http://IP:5601"
server.host: "0.0.0.0"
server.name: "node1"
server.port: 5601
elasticsearch.hosts: ["http://es地址:9200"]
i18n.locale: "zh-CN"
monitoring.ui.container.elasticsearch.enabled: true

#启动kibana
docker run --name=kibana -p 5601:5601 --restart=always -v/kibana/config:/usr/share/kibana/config -d kibana:7.16.3