k8s事件监控(通用版)

写了一个python脚本，然后做成k8s pod,用来监控k8s events:

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2022/1/16 2:21 下午
# @Author  : 5sdba
# @File    : webhook.py
# @Software: PyCharm

import time
import hmac
import hashlib
import base64
import urllib
import requests
from urllib import parse
from flask import Flask,request,jsonify,abort

app=Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def get_events():
    print("*********")

    if request.method == 'GET':
        return jsonify({'status': 'success'}), 200
    elif request.method == 'POST':
        sent_messange()
        return jsonify({'status': 'success'}), 200
    else:
        abort(400)


URL="https://oapi.dingtalk.com/robot/send?access_token=b6d4f153xxxxxxxxx"
secret = "xxxxxx"
def get_timestamp_sign():
    timestamp = str(round(time.time() * 1000))
    secret_enc = secret.encode('utf-8')
    string_to_sign = '{}\n{}'.format(timestamp, secret)
    string_to_sign_enc = string_to_sign.encode('utf-8')
    hmac_code = hmac.new(secret_enc, string_to_sign_enc,
                         digestmod=hashlib.sha256).digest()
    sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
    return (timestamp, sign)

def get_signed_url():
    timestamp, sign = get_timestamp_sign()
    webhook = URL + "&timestamp="+timestamp+"&sign="+sign
    return webhook

def get_webhook(mode):
    if mode == 0: # only 敏感字
       webhook = URL
    elif mode == 1 or  mode ==2 : # 敏感字和加签 或 # 敏感字+加签+ip
        webhook = get_signed_url()
    else:
        webhook = ""
        print("error! mode:   ",mode,"  webhook :  ",webhook)
    return webhook


def sent_messange():
    webhook = get_webhook(1)
    all_events = eval(str(request.get_data(),encoding="utf-8"))

    headers = {
        "Content-Type": "application/json; charset=UTF-8",
    }

    pagrem = {
        "msgtype": "markdown",
        "markdown": {
            "title": "K8s ops" + "....",
            "text": "<font color=#FF0000 size=3>操作详情：</font>" +
                    "\n\n>日志时间 ：" + str(all_events['EventTime']) +
                    "\n\n>pod组名：" + str(all_events['EventObject']) +
                    "\n\n>日志类型：" + str(all_events['EventReason']) +
                    "\n\n>日志级别：" + str(all_events['EventType']) +
                    "\n\n>日志详情：" + str(all_events['EventMessage'])
        },
        "at": {
            "atMobiles": [
                "123456"
            ]
        },
        "isAtAll": "False"
    }
    print(pagrem)
    r = requests.post(webhook, json = pagrem,headers=headers)

    print(r.status_code)
    print(r.text)


if __name__ == '__main__':
    app.run(host="0.0.0.0",port=5005, debug=True)

然后做成docker镜像，上传到自己的镜像仓库：

root@ip-10-10-xx-xx:~/webhook# tree .
.
├── Dockerfile
├── requirements.txt
└── webhook.pyc

root@ip-10-10-xxx-xxx:~/webhook# cat Dockerfile 

FROM python:3
LABEL org.opencontainers.image.authors="dba@xxxxx"
WORKDIR /
COPY ./requirements.txt /requirements.txt
RUN pip install --no-cache-dir -r requirements.txt
COPY . /
ENTRYPOINT [ "python" ]
CMD [ "webhook.py" ]

# docker build -t k8s_alerts:0.0.3 .
# docker run --name k8s_alerts -d -p 5000:5000 k8s_alerts:0.0.3
# docker commit -a "xxx" -m "k8s-notice" f71b18b3555d k8s_alerts:0.03
# docker tag k8s_alerts:0.03 dockerhub.fxxx/ops/k8s_alerts:0.03
# docker push dockerhub.xxxxxx.com/ops/k8s_alerts:0.03

然后利用阿里开源的kube-eventer来采集events时间发送到我这个pods(因为这个阿里的开源的感觉发送消息格式很难看，自定义也麻烦，因此只用来采集)，部署yaml如下：

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    name: kube-eventer
  name: kube-eventer
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-eventer
  template:
    metadata:
      labels:
        app: kube-eventer
     
    spec:
      dnsPolicy: ClusterFirstWithHostNet
      serviceAccount: kube-eventer
      containers:
        - image: registry.aliyuncs.com/acs/kube-eventer-amd64:v1.2.0-484d9cd-aliyun
          name: kube-eventer
          command:
            - "/kube-eventer"
            - "--source=kubernetes:https://kubernetes.default"
            ## .e.g,dingtalk sink demo 这个webhook 地址就是上面那个自定义发送的地址
            - --sink=webhook:http://10.x.x.x:5006/?level=Warning&custom_body_configmap=custom-webhook-body&custom_body_configmap_namespace=kube-system&method=POST&header=Content-Type=application/json

          env:
          # If TZ is assigned, set the TZ value as the time zone
          - name: TZ
            value: "Asia/Shanghai" 
          volumeMounts:
            - name: localtime
              mountPath: /etc/localtime
              readOnly: true
            - name: zoneinfo
              mountPath: /usr/share/zoneinfo
              readOnly: true
          resources:
            requests:
              cpu: 200m
              memory: 100Mi
            limits:
              cpu: 500m
              memory: 250Mi
      volumes:
        - name: localtime
          hostPath:
            path: /etc/localtime
        - name: zoneinfo
          hostPath:
            path: /usr/share/zoneinfo
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kube-eventer
rules:
  - apiGroups:
      - ""
    resources:
      - events
      - configmaps
    verbs:
      - get
      - list
      - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-eventer
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-eventer
subjects:
  - kind: ServiceAccount
    name: kube-eventer
    namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-eventer
  namespace: kube-system

---
apiVersion: v1
data:
  content: >-
    {"EventType": "{{ .Type }}",
    "EventKind": "{{ .InvolvedObject.Kind }}",
    "EventObject": "{{ .InvolvedObject.Name }}",
    "EventReason": "{{.Reason }}",
    "EventTime": "{{ .LastTimestamp }}",
    "EventMessage": "{{ .Message}}"}
kind: ConfigMap
metadata:
  name: custom-webhook-body
  namespace: kube-system

然后2个pod部署完毕以后，告警的日志如下：