filebeat-config.yaml
1 apiVersion: v1
2 data:
3 filebeat.yml: |-
4 filebeat.modules:
5 - module: system
6 filebeat.inputs:
7 - type: docker
8 multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3}'
9 multiline.negate: true
10 multiline.match: after
11 multiline.timeout: 15s
12 tags: ['prod-filebeat']
13 containers.ids:
14 - "*"
15 paths:
16 - /var/log/containers/*.log
17 processors:
18 - add_kubernetes_metadata:
19 in_cluster: true
20 symlinks: true
21 output.logstash:
22 hosts: ['logstash-http:5044']
23 logging.level: info
24 kind: ConfigMap
25 metadata:
26 creationTimestamp: "2020-03-04T08:10:09Z"
27 labels:
28 app: filebeat-config
29 k8s-app: filebeat
30 kubernetes.io/cluster-service: "true"
31 name: filebeat-config
32 namespace: logging
33 resourceVersion: "1477769"
34 selfLink: /api/v1/namespaces/logging/configmaps/filebeat-config
35 uid: bf737ab6-0d74-4860-8c86-fbca646a447e
logstah-custom-config.yaml
1 apiVersion: v1
2 data:
3 logstash.conf: |-
4 input {
5 beats {
6 port => 5044
7 }
8 }
9 filter {
10 if "prod-filebeat" in [tags]{
11 grok {
12 # 筛选过滤
13 match => {
14 "message" => "(?<access_time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d{3})\s+(?<level>\S+)\s+(?<process>\S+)\s+---\s+\[(?<thread>\S+)\](?<log>.*)"
15 }
16 # remove_field => ["message"]
17 }
18 # 过滤正则不匹配日志
19 if ![access_time] {
20 # 删除日志
21 drop {
22 }
23 }
24 if [level] !~ "(ERROR|WARN|INFO)"{
25 drop {}
26 }
27 }
28 }
29 output {
30 elasticsearch {
31 hosts => ["http://elasticsearch:9200"]
32 }
33 }
34 logstash.yml: |-
35 http.host: "0.0.0.0"
36 xpack.monitoring.enabled: true
37 xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
38 pipelines.yml: |-
39 - pipeline.id: main
40 path.config: "/usr/share/logstash/pipeline"
41 queue.type: persisted
42 kind: ConfigMap
43 metadata:
44 creationTimestamp: "2020-03-04T08:16:50Z"
45 name: logstah-custom-config
46 namespace: logging
47 resourceVersion: "2195747"
48 selfLink: /api/v1/namespaces/logging/configmaps/logstah-custom-config
49 uid: ee0fddb1-827e-4dca-9149-1595c9b39713
