ELK 收集 K8S (containerd 容器运行时) 三

部署 logstash

mkdir -p /data/yaml/k8s-logging/logstash
cd /data/yaml/k8s-logging/logstash

cat cm.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-config
  namespace: k8s-logging
  labels:
    elastic-app: logstash
data:
  logstash.conf: |-
    input {
        kafka {
                bootstrap_servers => "kafka-svc:9092"
                group_id => "services"
                consumer_threads => 5
                decorate_events => true
                topics_pattern => "(k8s|docker)-.*"
                auto_offset_reset => "latest"
                codec => json { charset => "UTF-8" }
        }

    }

    filter {
      mutate {
        remove_field => [ "@version", "stream", "container", "agent", "log", "host", "input", "ecs" ]
      }
    }

    ## Add your filters / logstash plugins configuration here

    output {
        elasticsearch {
                hosts => "elasticsearch:9200"
                user => "elastic"
                password => "changeme"
                index =>  "%{[@metadata][topic]}-%{+YYYY-MM-dd}"
        }
    }

kubectl apply -f cm.yaml

cat deployment.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: logstash
  namespace: k8s-logging
spec:
  replicas: 1
  selector:
    matchLabels:
      elastic-app: logstash
  template:
    metadata:
      labels:
        elastic-app: logstash
    spec:
      containers:
      - name: logstash
        image: logstash:7.4.2
        env:
        - name: "PIPELINE_WORKERS" # 提高 logstash 消费 kafka 消息的能力，减少延迟
          value: "8"
        - name: "PIPELINE_BATCH_SIZE"
          value: "5000"
        - name: "PIPELINE_BATCH_DELAY"
          value: "10"
        volumeMounts:
        - name: config
          mountPath: /usr/share/logstash/pipeline/logstash.conf
          readOnly: true
          subPath: logstash.conf
        - mountPath: /etc/localtime
          readOnly: true 
          name: time-data
      volumes: 
      - name: config
        configMap:
          name: logstash-config
      - name: time-data 
        hostPath: 
          path: /usr/share/zoneinfo/Asia/Shanghai

kubectl apply -f deployment.yaml 

部署 kibana

mkdir -p /data/yaml/k8s-logging/kibana
cd /data/yaml/k8s-logging/kibana

cat deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: k8s-logging
spec:
  replicas: 1
  selector:
    matchLabels:
      elastic-app: kibana
  template:
    metadata:
      labels:
        elastic-app: kibana
    spec:
      containers:
      - name: kibana
        image: kibana:7.4.2
        ports:
        - containerPort: 5601
          protocol: TCP
        env:
        - name: "ELASTICSEARCH_URL"
          value: "http://elasticsearch:9200"
        - name: "ELASTICSEARCH_REQUESTTIMEOUT"
          value: "120000"
        volumeMounts: 
        - mountPath: /etc/localtime
          readOnly: true 
          name: time-data
      volumes: 
      - name: time-data 
        hostPath: 
          path: /usr/share/zoneinfo/Asia/Shanghai

kubectl apply -f deployment.yaml 

cat svc.yaml 
kind: Service
apiVersion: v1
metadata:
  labels:
    elastic-app: kibana
  name: kibana-service
  namespace: k8s-logging
spec:
  ports:
    - port: 5601
      targetPort: 5601
      nodePort: 30008
  selector:
    elastic-app: kibana
  type: NodePort

kubectl apply -f svc.yaml 

访问