filebeat实战
1.打开filebeat支持nginx模块
[root@es-node1 /etc/filebeat]#ls
fields.yml filebeat.reference.yml filebeat.yml filebeat.yml.bak modules.d
[root@es-node1 /etc/filebeat]#ls modules.d/
[root@es-node1 /etc/filebeat]#filebeat version
filebeat version 7.9.1 (amd64), libbeat 7.9.1 [ad823eca4cc74439d1a44351c596c12ab51054f5 built 2020-09-01 19:58:51 +0000 UTC]
[root@es-node1 /etc/filebeat]#
查看开启了哪些模块
[root@es-node1 /etc/filebeat]#filebeat modules list
Error in modules manager: modules management requires 'filebeat.config.modules.path' setting
修改配置文件
[root@es-node1 /etc/filebeat]#tail -5 /etc/filebeat/filebeat.yml
logging.to_files: true
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
默认未激活的模块
[root@es-node1 /etc/filebeat]#filebeat modules list|wc -l
60
激活filebeat对nginx支持的模块,本质就是去修改modules.d/下的yml文件,去掉了disable后缀
[root@es-node1 /etc/filebeat]#filebeat modules enable nginx
Enabled nginx
# 打开nginx模块文件,添加nginx日志输入源
cat > /etc/filebeat/modules.d/nginx.yml << 'EOF'
- module: nginx
access:
enabled: true
var.paths: ["/var/log/nginx/access.log"]
error:
enabled: true
var.paths: ["/var/log/nginx/error.log"]
ingress_controller:
enabled: false
EOF
# 最后修改filebeat.yml,不需要input了
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: true
output.elasticsearch:
hosts: ["10.0.0.51:9200"]
indices:
- index: "nginx-access-%{[agent.version]}-%{+yyyy.MM}"
when.contains:
log.file.path: "/var/log/nginx/access.log"
- index: "nginx-error-%{[agent.version]}-%{+yyyy.MM}"
when.contains:
log.file.path: "/var/log/nginx/error.log"
setup.ilm.enabled: false
setup.template.enabled: false
logging.level: info
logging.to_files: true
# 清理es、filebeat、nginx环境、测试使用
# 重启filebeat
查看kibana(pipeline)
kibana展示es
2.filebeat收集tomcat日志
# jdk
[root@tomcat-10 ~/tomcat-all]#tar -xf jdk-8u221-linux-x64.tar.gz -C /opt
[root@tomcat-10 ~/tomcat-all]#ls /opt
jdk1.8.0_221
配置软连接,修改PATH
[root@tomcat-10 ~/tomcat-all]#ln -s /opt/jdk1.8.0_221/ /opt/jdk8
# sed命令,快速修改 /etc/profile 全局用户环境变量文件
sed -i.ori '$a export JAVA_HOME=/opt/jdk8\nexport PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH\nexport CLASSPATH=.$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/lib/tools.jar' /etc/profile
# tomcat
[root@es-node1 /opt]#tar -xf apache-tomcat-8.0.27.tar.gz
[root@es-node1 /opt]#cd /opt/apache-tomcat-8.0.27/
[root@es-node1 /opt/apache-tomcat-8.0.27]#
# 修改tomcat日志格式
135 <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
136 prefix="localhost_access_log" suffix=".txt"
137 pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","m ethod":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i",&qu ot;AgentVersion":"%{User-Agent}i"}"/>
[root@es-node1 /opt/jdk8]#/opt/apache-tomcat-8.0.27/bin/shutdown.sh
Using CATALINA_BASE: /opt/apache-tomcat-8.0.27
Using CATALINA_HOME: /opt/apache-tomcat-8.0.27
Using CATALINA_TMPDIR: /opt/apache-tomcat-8.0.27/temp
Using JRE_HOME: /opt/jdk8
Using CLASSPATH: /opt/apache-tomcat-8.0.27/bin/bootstrap.jar:/opt/apache-tomcat-8.0.27/bin/tomcat-juli.jar
[root@es-node1 /opt/jdk8]#/opt/apache-tomcat-8.0.27/bin/startup.sh
Using CATALINA_BASE: /opt/apache-tomcat-8.0.27
Using CATALINA_HOME: /opt/apache-tomcat-8.0.27
Using CATALINA_TMPDIR: /opt/apache-tomcat-8.0.27/temp
Using JRE_HOME: /opt/jdk8
Using CLASSPATH: /opt/apache-tomcat-8.0.27/bin/bootstrap.jar:/opt/apache-tomcat-8.0.27/bin/tomcat-juli.jar
Tomcat started.
[root@es-node1 /opt/jdk8]#
# 查看tomcat日志。是否是json
[root@es-node1 /opt/jdk8]#cat /opt/apache-tomcat-8.0.27/logs/localhost_access_log.2023-02-19.txt
{"clientip":"127.0.0.1","ClientUser":"-","authenticated":"-","AccessTime":"[19/Feb/2023:20:55:36 +0800]","method":"GET / HTTP/1.1","status":"200","SendBytes":"11250","Query?string":"","partner":"-","AgentVersion":"curl/7.29.0"}
[root@es-node1 /opt/jdk8]#
#配置filebeat文件
Filebeat.yml
1 filebeat.inputs:
2 - type: log
3 enabled: true
4 paths:
5 - /opt/apache-tomcat-8.0.27/logs/localhost_access_log.*.txt
6 json.keys_under_root: true
7 json.overwrite_keys: true
8 tags: ["tomcat"]
9
10 filebeat.config.modules:
11 path: ${path.config}/modules.d/*.yml
12 reload.enabled: true
13
14 output.elasticsearch:
15 hosts: ["10.0.0.18:9200"]
16 indices:
17 - index: "nginx-access-%{[agent.version]}-%{+yyyy.MM}"
18 when.contains:
19 log.file.path: "/var/log/nginx/access.log"
20
21 - index: "nginx-error-%{[agent.version]}-%{+yyyy.MM}"
22 when.contains:
23 log.file.path: "/var/log/nginx/error.log"
24 - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM}"
25 when.contains:
26 tags: "tomcat"
27
28 setup.ilm.enabled: false
29 setup.template.enabled: false
30
31 logging.level: info
32 logging.to_files: true
重启filebeat
[root@es-node1 /opt/jdk8]#vim /etc/filebeat/filebeat.yml
[root@es-node1 /opt/jdk8]#systemctl restart filebeat.service
3.小结
ElasticSearch + Filebeat + kibana玩法先到这,基本流程
1. filebeat + 目标软件日志(json)
2. 发给es
3. kibana管理es索引,进行数据展示,过滤,可视化管理
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· DeepSeek “源神”启动!「GitHub 热点速览」
· 微软正式发布.NET 10 Preview 1:开启下一代开发框架新篇章
· 我与微信审核的“相爱相杀”看个人小程序副业
· C# 集成 DeepSeek 模型实现 AI 私有化(本地部署与 API 调用教程)
· DeepSeek R1 简明指南:架构、训练、本地部署及硬件要求