日志收集案例-容器内置日志收集

  引用tomcat镜像构建filebeat镜像

root@deploy:/dockerfile/project/app-filebeat/ cat Dockerfile 
FROM harbor.cncf.net/web/tomcat:8.5.43

ADD filebeat-7.12.1-amd64.deb /tmp
RUN cd /tmp/ && dpkg -i filebeat-7.12.1-amd64.deb
ADD filebeat.yml /etc/filebeat/filebeat.yml
ADD filebeat.sh /
ENTRYPOINT ["/filebeat.sh"]

 

  创建filebeat启动脚本

root@deploy:/dockerfile/project/app-filebeat/ cat filebeat.sh 
#!/bin/bash
/usr/local/tomcat/bin/catalina.sh start
sleep 3s
/usr/share/filebeat/bin/filebeat -e -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat &
tail -f /usr/local/tomcat/logs/catalina.out

 

  创建filebeat服务配置文件

root@deploy:/dockerfile/project/app-filebeat/ cat filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/catalina.out   #获取tomat catalina.out日志
  tags: ["catalinalog"]
  multiline.pattern: '^\d{2}' #\d匹配数字开头，数字为2位
  multiline.negate: true
  multiline.match: after
  multiline.max_lines: 10000   #最大的合并行数，默认合并的数量为500

  
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/localhost_access_log.*.txt  #获取tomcat访问日志
  tags: ["accesslog"]
  json.keys_under_root: true   #Flase会将json解析的格式存储至messages，改为true则不存储至message
  json.overwrite_keys: true   #覆盖默认message字段，使用自定义json格式的key

 
output.kafka:
  hosts: ["192.168.100.103:9092","192.168.100.102:9092","192.168.100.103:9092"]
  topic: "tomcatlog"
  partition.round_robin:
    reachable_only: false
  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

 

  创建tomcat容器k8s yaml

root@deploy:/dockerfile/project/app-filebeat/ cat tomcat-filebeat.yml 
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    app: tomcat
  name: tomcat-deployment
  namespace: test
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tomcat
  template:
    metadata:
      labels:
        app: tomcat
    spec:
      containers:
      - name: tomcat
        image: harbor.cncf.net/project/tomcat-app1:1.0
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          protocol: TCP
          name: http
        resources:
          limits:
            cpu: 1
            memory: "512Mi"
          requests:
            cpu: 500m
            memory: "512Mi"

 

  创建tomcat容器svc

root@deploy:/dockerfile/project/app-filebeat/ cat tomcat-service.yaml 
---
kind: Service
apiVersion: v1
metadata:
  labels:
    app: tomcat
  name: tomcat
  namespace: test
spec:
  type: NodePort
  ports:
  - name: http
    port: 80
    protocol: TCP
    targetPort: 8080
    nodePort: 30180
  selector:
    app: tomcat

 

  浏览器访问：

      

   kafkatools查看主题消息

      

   12、配置logstash

[root@logstash ~]/ vim /etc/logstash/conf.d/tomcat.conf
input {
  kafka {
    bootstrap_servers => "192.168.100.101:9092,192.168.100.102:9092,192.168.100.103:9092"   #生产者kafka地址
    topics => ["tomcatlog"]             #消费主题
    codec => "json"
  }
}

output {
    if "catalinalog" in [tags] {
      elasticsearch {
        hosts => ["http://192.168.100.70:9200","192.168.100.71:9200","192.168.100.72:9200"]
        manage_template => false
        index => "tomcat-catalinalog-%{+yyyy.MM.dd}"
      }
    }
    if "accesslog" in [tags] {
      elasticsearch {
        hosts => ["http://192.168.100.70:9200","192.168.100.71:9200","192.168.100.72:9200"]
        manage_template => false
        index => "tomcat-accesslog-%{+yyyy.MM.dd}"
      }
    }
}

 

       重启logstash

[root@logstash ~]/ systemctl restart logstash

 

  查看ES，两个类型的日志索引

      

  13、配置kibana创建索引模式

  查看访问日志

      

  查看catalina日志