4.filebeat

安装：下载好安装包，rpm安装就行了
?
1
rpm -ivh filebeat-6.8.8-x86_64.rpm
  • 收集nginx访问日志和错误日志（转JSON日志时，如果不成功，kibana在创建索引时候message字段拆不开）　　
?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
#更改nginx日志格式为JSON，注意逗号
    log_format  json '{"@timestamp":"$time_iso8601",'
                           '"@version":"1",'
                           '"client":"$remote_addr",'
                           '"url":"$uri",'
                           '"status":"$status",'
                           '"domain":"$host",'
                           '"host":"$server_addr",'
                           '"size":$body_bytes_sent,'
                           '"responsetime":$request_time,'
                           '"referer": "$http_referer",'
                           '"ua": "$http_user_agent"'
               '}';
    access_log  /var/log/nginx/access.log  json;
 
#filebeat配置
[root@nginx ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true  #解析json日志用的选项
  json.overwrite_keys: true
  tags: ["access"]
 
- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
 
setup.kibana:
  host: "192.168.1.75:5601"
 
output.elasticsearch:
  hosts: ["192.168.1.75:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"
 
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
  • 收集tomcat日志和nginx日志
?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
#更改tomcat访问日志格式
#vim server.xml
<Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="localhost_access_log" suffix=".txt"
               pattern="{"clientip":"%h","ClientUser":"%l","authenticated":"%u","AccessTime":"%t","method":"%r","status":"%s","SendBytes":"%b","Query?string":"%q","partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>
 
#更改filebeat配置文件
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]
 
- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
 
- type: log
  enable: true
  paths:
    - /root/apache-tomcat-8.5.37/logs/localhost_access_log.*
  tags: ["tomcat"]
  json.keys_under_root: true
  json.overwrite_keys: true
 
setup.kibana:
  host: "192.168.1.75:5601"
 
output.elasticsearch:
  hosts: ["192.168.1.75:9200"]
  #index: "nginx-%{[beat.version]}-%{+yyyy.MM}"
  indices:
    - index: "nginx-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "access"
    - index: "nginx-error-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "error"
    - index: "tomcat-access-%{[beat.version]}-%{+yyyy.MM}"
      when.contains:
        tags: "tomcat"
 
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
  • 收集java日志，多行合并成一行
?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/elasticsearch/elasticsearch.log
  multiline.pattern: '^\['    #匹配[开头的行
  multiline.negate: true      #与正则不匹配的行合并成一行
  multiline.match: after      #将negate匹配到的行向上或向下合并成一行。
 
setup.kibana:
  host: "10.0.0.51:5601"
 
output.elasticsearch:
  hosts: ["10.0.0.51:9200"]
  index: "es-java-%{[beat.version]}-%{+yyyy.MM}"
 
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.enabled: false
setup.template.overwrite: true
  • 使用模块收集Nginx日志
?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
1.配置filebeat主配置文件
[root@localhost ~]# egrep -v '^$|#' /etc/filebeat/filebeat.yml
filebeat.config.modules:    #开启模块
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
setup.kibana:
  host: "192.168.2.100:5601"
output.elasticsearch:
  hosts: ["192.168.2.100:9200"]
  indices:
    - index: "nginx_access-%{[beat.version]}-%{+yyyy.MM.dd}"
      when.contains:
        fileset.name: "access"
    - index: "nginx_error-%{[beat.version]}-%{+yyyy.MM.dd}"
      when.contains:
        fileset.name: "error"
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
 
2.激活并配置filebeat的nginx模块
[root@localhost ~]# filebeat modules enable nginx
[root@localhost ~]# vim /etc/filebeat/modules.d/nginx.yml
- module: nginx
  access:
    enabled: true
    var.paths: ["/var/log/nginx/access.log"]
  error:
    enabled: true
    var.paths: ["/var/log/nginx/error.log"]
 
3.将nginx日志更改为普通模式
4.es上安装两个插件，es6.7版本之后就不用安装了，自带这两个插件
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-user-agent-6.6.0.zip
/usr/share/elasticsearch/bin/elasticsearch-plugin install file:///root/ingest-geoip-6.6.0.zip
5.重启es
6.重启filebeat
  • 中间加一层redis，架构如下图

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#需要更改filebeat和logstash的配置文件
#filebeat.yml
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: true
 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true  #解析json日志用的选项
  json.overwrite_keys: true
  tags: ["access"]
 
- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]
 
setup.kibana:
  host: "192.168.2.100:5601"
 
output.redis:  #这次发送给redis，不用区分开存到redis不同的键，因为在上面已经插入了tags，取值得时候根据tags做判断
  hosts: ["192.168.2.4"]
  key: "filebeat"
  #password: "password"
  db: 0
  timeout: 5
 
setup.template.name: "nginx"
setup.template.pattern: "nginx_*"
setup.template.enabled: false
setup.template.overwrite: true
 
##下面是logstash的配置，启动的时候需要指定logstash -f xxx/logstsh_redis.conf
##[root@localhost ~]# vim /etc/logstash/conf.d/logstash_redis.conf
input {
  redis {
    host => "192.168.2.4"
    port => "6379"
    db => "0"
    key => "filebeat"    #读这个redis键
    data_type => "list"
  }
}
 
filter {
  mutate {  #这两项是将nginx日志里面的两项反馈时长转换为浮点数，后期可以做比较
    convert => ["upstream_time", "float"]
    convert => ["request_time", "float"]
  }
}
 
output {
    stdout {}
     if "access" in [tags] {   #判断在filebeat中插入的键值，对应存到es的索引中
      elasticsearch {
        hosts => "http://192.168.2.100:9200"
        manage_template => false
        index => "nginx_access-%{+yyyy.MM.dd}"
      }
    }
    if "error" in [tags] {
      elasticsearch {
        hosts => "http://192.168.2.100:9200"
        manage_template => false
        index => "nginx_error-%{+yyyy.MM.dd}"
      }
    }
}
  • 使用keepalived+nginx给redis做高可用，架构图如下，这里面唯一不同的地方就是filebeat发送地址和logstash接收地址都改为nginx的vip

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#下面这段配置需要放在http段之外，因为属于4层带端口负载均衡，为了保证数据一致性，给一台redis后端主机加上backup，当主的挂了才使用备的。
[root@lb02 ~]# cat /etc/nginx/nginx.conf
..........................
stream {
  upstream redis {
      server 10.0.0.51:6381 max_fails=2 fail_timeout=10s;
      server 10.0.0.51:6382 max_fails=2 fail_timeout=10s backup;
  }
   
  server {
          listen 6379;
          proxy_connect_timeout 1s;
          proxy_timeout 3s;
          proxy_pass redis;
  }
}
  • 收集docker日志，未测试
posted @   ForLivetoLearn  阅读(220)  评论(0)  编辑  收藏  举报
编辑推荐：
· Linux系列：如何用heaptrack跟踪.NET程序的非托管内存泄露
· 开发者必知的日志记录最佳实践
· SQL Server 2025 AI相关能力初探
· Linux系列：如何用 C#调用 C方法造成内存泄露
· AI与.NET技术实操系列（二）：开始使用ML.NET
阅读排行：
· 没有Manus邀请码？试试免邀请码的MGX或者开源的OpenManus吧
· 无需6万激活码！GitHub神秘组织3小时极速复刻Manus，手把手教你使用OpenManus搭建本
· C#/.NET/.NET Core优秀项目和框架2025年2月简报
· DeepSeek在M芯片Mac上本地化部署
· 葡萄城 AI 搜索升级：DeepSeek 加持，客户体验更智能
点击右上角即可分享
微信分享提示