使用 python 收集 kubernetes events 并写入 elasticsearch

from kubernetes import client, config, watch
from elasticsearch import Elasticsearch
import arrow
import sys
import requests
import json

dingding_webhook = "https://oapi.dingtalk.com/robot/send?access_token=xxxxx"

hosts = [
    '172.16.21.39:9200',
    '172.16.21.40:9200',
    '172.16.21.41:9200'
]

def send_text(content):
    data = {
        "msgtype": "text",
        "text": {
            "content": content
        }
    }
    requests.post(url=dingding_webhook, json=data)


now = arrow.now('Asia/Shanghai')
index = "kube-events"
index_today = "{}.{}".format(index, now.format("YYYY.MM.DD"))

cluster_name = sys.argv[1]
kube_config = sys.argv[2]

with Elasticsearch(hosts=hosts, timeout=120) as es:
    if not es.indices.exists(index=index_today):
        es.indices.create(index=index_today)

    # Configs can be set in Configuration class directly or using helper utility
    config.load_kube_config(config_file=kube_config)
    v1 = client.CoreV1Api()
    w = watch.Watch()

    try:
        for event in w.stream(v1.list_event_for_all_namespaces):
            doc = {
                "cluster_name": cluster_name,
                "time_iso8601": arrow.Arrow.fromdatetime(event['object'].metadata.creation_timestamp).isoformat(),
                "namespace": event['object'].metadata.namespace,
                "type": event['object'].type,
                "reason": event['object'].reason,
                "message": event['object'].message
            }
            # print(json.dumps(doc))
            # if doc["type"] == "Warning":
            #     send_text('[ {} ]- {}'.format(cluster_name, doc["message"]))
            if es.exists(index=index_today, doc_type=index, id=event['object'].metadata.uid):
                continue
            es.create(index=index_today, doc_type=index, body=doc, id=event['object'].metadata.uid)
    except Exception as e:
        pass

    w.stop()

使用 supervisord 将程序后台运行。

[program:ali-14]
command = /usr/local/bin/python3.6 /ops/scripts/kube_events.py ali-14 /ops/k8s_config/ali-14
autorestart = true
redirect_stderr = true
stdout_logfile = /ops/logs/ali-14.log
stopasgroup = true
[program:m7]
command = /usr/local/bin/python3.9 /ops/scripts/kube_events.py m7 /ops/k8s_config/m7
autorestart = true
redirect_stderr = true
stdout_logfile = /ops/logs/m7.log
stopasgroup = true
[program:cm]
command = /usr/local/bin/python3.9 /ops/scripts/kube_events.py cm /ops/k8s_config/cm
autorestart = true
redirect_stderr = true
stdout_logfile = /ops/logs/cm.log
stopasgroup = true
[program:meteo]
command = /usr/local/bin/python3.6 /ops/scripts/kube_events.py meteo /ops/k8s_config/meteo
autorestart = true
redirect_stderr = true
stdout_logfile = /ops/logs/meteo.log
stopasgroup = true

写入 ES 之后就可以通过 kibana 查看了。