k8s 结合filebeat及logstash日志收集

日志收集流程

filebeat--->kafka--->logstash--->elasticsearch

日志收集模式

具有日志代理功能的sidecar模式收集日志

logstash.conf

input {
  kafka {
    bootstrap_servers => "172.16.3.213:19092,172.16.3.213:29092,172.16.3.213:39092"
    topics => ["app-1"]
    codec => "json"
  }
}

output {
  if [fields][source] == "app1" {
    elasticsearch {
      hosts => ["https://172.16.3.9:9200"]
      index => "app1-%{+YYYY.MM.dd}"
      user => "xxxxxxx"
      password => "xxxxxxx"
      cacert => "/usr/share/logstash/config/ca.crt" 
    }
  }

}

filebeat-app1-conf.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeat-app1-config
  namespace: demoapp
data:
  filebeat.yml: |
    filebeat.inputs:
    - type: filestream
      id: app1
      enabled: true
      paths:
        - /var/log/1.log
      fields:
        source: app1

    filebeat.config.modules:
      path: ${path.config}/modules.d/*.yml
      reload.enabled: false

    processors:
    - add_host_metadata:
        when.not.contains.tags: forwarded
    #- add_cloud_metadata: ~
    #- add_docker_metadata: ~
    #- add_kubernetes_metadata: ~

    output.kafka:
      hosts: ["172.16.3.213:19092", "172.16.3.213:29092", "172.16.3.213:39092"]
      topic: 'app-1'   
      partition.round_robin:
        reachable_only: false

      required_acks: 1
      compression: gzip
      max_message_bytes: 1000000

demo-ns.yaml

apiVersion: v1
kind: Namespace
metadata:
    name: demoapp

depoly-demo.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: demoapp-deployment
  namespace: demoapp
spec:
  replicas: 1
  selector:
    matchLabels:
      app: demoapp
      controller: demoapp
  template:
    metadata:
      labels:
        app: demoapp
        controller: demoapp
    spec:
      containers:
      - name: demoapp
        image: busybox:1.28
        ports:
        - containerPort: 80
          name: http
        args:
        - /bin/sh
        - -c
        - >
          i=0;
          while true;
          do
            echo "$i: $(date)" >> /var/log/1.log;
            echo "$(date) INFO $i" >> /var/log/2.log;
            i=$((i+1));
            sleep 1;
          done      
        resources:
          requests:
            memory: "256Mi"
            cpu: "50m"
          limits:
            memory: "256Mi"
            cpu: "50m"
        volumeMounts:
        - name: varlog
          mountPath: /var/log	
      - name: filebeat
        image: docker.elastic.co/beats/filebeat:8.7.0
        resources:
          requests:
            memory: "256Mi"
            cpu: "50m"
          limits:
            memory: "256Mi"
            cpu: "50m"
        volumeMounts:
        - name: varlog
          mountPath: /var/log	
        - name: config-volume
          mountPath: /usr/share/filebeat/filebeat.yml
          subPath: filebeat.yml
      volumes:
      - name: varlog
        emptyDir: {}
      - name: config-volume
        configMap:
          name: filebeat-app1-config


---
apiVersion: v1
kind: Service
metadata:
  name: demoapp-svc
  namespace: demoapp
spec:
  selector:
    app: demoapp
    controller: demoapp
  ports:
  - name: http
    port: 80
    targetPort: 80

创建demo资源

# kubectl apply -f demo-ns.yaml -f filebeat-app1-conf.yaml -f depoly-demo.yaml

查看kafka数据

查看es索引

查看日志信息