Monitoring-K8s资源观测工具Kubewatch

一、Kubewatch介绍

Kubewatch是一款Kubernetes 资源观测工具，实时监控 Kubernetes 集群中各种资源的新建、更新和删除，并实时通知到各种协作软件/聊天软件，目前支持的通知渠道有：
- slack
- hipchat
- mattermost
- flock
- webhook

二、Helm部署kubewatch

官网github（bitnami）：https://github.com/bitnami-labs/kubewatch
调配 chart values ：监视的ns、nodeSelector、webhook url

helm3 部署

helm install kubewatch ./kubewatch --set='rbac.create=true,resourcesToWatch.pod=true,resourcesToWatch.daemonset=true' --namespace monitoring
helm3 uninstall kubewatch --namespace monitoring

# kubectl get pods -n monitoring
kubewatch-5994fbcb59-qt9cz                1/1     Running   0          20h

三、Webhook 配置例子

我这里使用Python Flask编写了一个 webhook proxy 将消息格式化并过滤后推送至Worktile；
代码仅作为范例参考，也可直接推送到DingDing ，参考地址：https://segmentfault.com/a/1190000021219010?utm_source=tag-newest

# cat hook-server/flask-message.py
#!/usr/bin/python3
from flask import Flask
from flask import request, jsonify, abort
import datetime
import json
import requests

# 日志
import logging
from logging import handlers

class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射

    def __init__(self, filename, level='info', when='D', backCount=3,
                 fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
        self.logger = logging.getLogger(filename)
        format_str = logging.Formatter(fmt)  # 设置日志格式
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(format_str)  # 设置屏幕上显示的格式
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
                                               encoding='utf-8')  # 往文件里写入#指定间隔时间自动生成文件的处理器
        # midnight 每天凌晨
        th.setFormatter(format_str)  # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)

# worktile webhook
wt_webhook_url = 'https://hook.toptops.com/incoming/asdasdnashjdbasjdbasjdbajsd'
def worktile_webhook(hookurl=wt_webhook_url, string=None):
    print('send message to worktile webhook ')
    headers = {
        "Content-Type": "application/json; charset=UTF-8",
        # "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
    }

    Data = {
        "text": string,
        # "safe": "0"
    }

    print(Data, 'worktile')
    response = requests.post(hookurl, data=json.dumps(Data), headers=headers).text
    print(response)


# 字段格式化
class Json_sik(object):
    @staticmethod
    def js(payload):
        # print(payload)
        """
        format :
            pod - 删除
            时间 - 2021-03-17T10:18:24
            空间 - default
            详情 - A `pod` in namespace `` has been `updated`:`default/web-1`
        """
        # 格式化字符串
        kind = payload['eventmeta']['kind']
        reason = payload['eventmeta']['reason']
        # time = payload['time']
        time = datetime.datetime.now().replace(microsecond=0)
        namespace = payload['eventmeta']['name']
        msg = payload['text']

        if str(reason) == str("deleted") or str(reason) == str("created"):
            # send_text=
            text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
            # print(text)
            worktile_webhook(hookurl=wt_webhook_url, string=text)
            return text

app = Flask(__name__)

@app.route('/kubewatch', methods=['GET', 'POST'])
def webhook():
    if request.method == 'GET':
        logging.debug('Get Method not implemented')
        return jsonify({'status': 'success'}), 200

    elif request.method == 'POST':
        payload = request.json
        sim = Json_sik().js(payload=payload)
        return jsonify({'status': 'success'}), 200

    else:
        abort(400)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port='10010')

测试，可以临时删除一个监视的ns下的pod，查看kubewatch日志，是否获取到事件

time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-rd-admin-prod-6d499bf6df-ml9sf" pkg=kubewatch-pod
time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/pc-jira-importer-sky-prod-886b6ff65-wbkh7" pkg=kubewatch-pod
time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-sky-portal-prod-8476b457bf-lcs8h" pkg=kubewatch-pod
time="2021-03-18T11:27:33Z" level=info msg="Processing add to pod: prod/wt-rd-devops-rpc-prod-5fd46465d8-9562d" pkg=kubewatch-pod

优化消息冗余过滤，仅有资源删除、创建时推送消息；

from flask import Flask
from flask import request, jsonify, abort

# import simplejson
# import subprocess
import datetime
import json
import requests
import re

# 日志
import logging
from logging import handlers

class Logger(object):
    level_relations = {
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'crit': logging.CRITICAL
    }  # 日志级别关系映射

    def __init__(self, filename, level='info', when='D', backCount=3,
                 fmt='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'):
        self.logger = logging.getLogger(filename)
        format_str = logging.Formatter(fmt)  # 设置日志格式
        self.logger.setLevel(self.level_relations.get(level))  # 设置日志级别
        sh = logging.StreamHandler()  # 往屏幕上输出
        sh.setFormatter(format_str)  # 设置屏幕上显示的格式
        th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount,
                                               encoding='utf-8')  # 往文件里写入#指定间隔时间自动生成文件的处理器
        # 实例化TimedRotatingFileHandler
        # interval是时间间隔，backupCount是备份文件的个数，如果超过这个个数，就会自动删除，when是间隔的时间单位，单位有以下几种：
        # S 秒
        # M 分
        # H 小时、
        # D 天、
        # W 每星期（interval==0时代表星期一）
        # midnight 每天凌晨
        th.setFormatter(format_str)  # 设置文件里写入的格式
        self.logger.addHandler(sh)  # 把对象加到logger里
        self.logger.addHandler(th)


# worktile webhook
wt_webhook_url = 'https://hook.toptops.com/incoming/asdasdnashjdbasjdbasjdbajsd'
def worktile_webhook(hookurl=wt_webhook_url, string=None):
    print('send message to worktile webhook ')
    headers = {
        "Content-Type": "application/json; charset=UTF-8",
        # "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Safari/537.36",
    }

    Data = {
        "text": string,
        # "safe": "0"
    }

    print(Data, 'worktile')
    response = requests.post(hookurl, data=json.dumps(Data), headers=headers).text
    print(response)


create_pods = []
delete_pods = []

# 字段格式化
class Json_sik(object):
    @staticmethod
    def js(payload):
        # print(payload)
        """
        format :
            pod - 删除
            时间 - 2021-03-17T10:18:24
            空间 - default
            详情 - A `pod` in namespace `` has been `updated`:`default/web-1`
        """
        # print(payload)

        global create_pods, delete_pods

        # 格式化字符串
        kind = payload['eventmeta']['kind']
        reason = payload['eventmeta']['reason']
        # time = payload['time']
        time = datetime.datetime.now().replace(microsecond=0)
        namespace = payload['eventmeta']['name']
        msg = payload['text']

        # print(type(reason), reason)


        if str(reason) == str("deleted") or str(reason) == str("created"):

            # for i in ns_and_pod_list:
            #     if i not in new_list:
            #         new_list.append(i)
            # print(new_list)
            # print(ns_and_pod_list, 'ns_and_pod_list 列表展示')

            print(payload)
            if str(reason) == str("created"):
                try:
                    print('1~~~~~~~')
                    reason = payload['eventmeta']['reason']  # 动作
                    msg_str = payload['text']
                    ns_and_pod = msg_str[msg_str.rfind('\n'):]
                    ns_and_pod = re.sub(r'[:/\\?*“”<>|`/\n]', '', ns_and_pod)
                    new_str = '{}-{}'.format(ns_and_pod, reason)
                    list = (new_str.split('-'))
                    new_str_c = '{}-{}-{}'.format(list[0], list[1], list[2])
                    create_pods.append(new_str_c)
                except Exception as e:
                    print('2~~~~~~~~')
                    reason = payload['eventmeta']['reason']  # 动作
                    msg_str = payload['text']
                    ns_and_pod = msg_str[msg_str.rfind('\n'):]
                    ns_and_pod = re.sub(r'[:/\\?*“”<>|`/\n]', '', ns_and_pod)
                    new_str_c = '{}-{}'.format(ns_and_pod, reason)
                    create_pods.append(new_str_c)
                print(create_pods)
                if len(create_pods) < 1:
                    print(create_pods, '创建pods')
                    text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
                    worktile_webhook(hookurl=wt_webhook_url, string=text)
                    return text
                elif create_pods.count(new_str_c) <= 1:
                    print(create_pods, '创建pods')
                    text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
                    worktile_webhook(hookurl=wt_webhook_url, string=text)
                    return text
            else:
                try:
                    print('3~~~~~~~')
                    reason = payload['eventmeta']['reason']  # 动作
                    msg_str = payload['text']
                    ns_and_pod = msg_str[msg_str.rfind('/'):]
                    ns_and_pod = re.sub(r'[:/\\?*“”<>|`]', '', ns_and_pod)
                    new_str = '{}-{}'.format(ns_and_pod, reason)
                    list = (new_str.split('-'))
                    new_str_d = '{}-{}-{}'.format(list[0], list[1], list[2])
                    delete_pods.append(new_str_d)
                except Exception as e:
                    print('4~~~~~~~')
                    reason = payload['eventmeta']['reason']  # 动作
                    msg_str = payload['text']
                    ns_and_pod = msg_str[msg_str.rfind('/'):]
                    ns_and_pod = re.sub(r'[:/\\?*“”<>|`]', '', ns_and_pod)
                    new_str_d = '{}-{}'.format(ns_and_pod, reason)
                    delete_pods.append(new_str_d)
                print(delete_pods)
                if len(delete_pods) < 1:
                    print(delete_pods, '删除pods')
                    text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
                    worktile_webhook(hookurl=wt_webhook_url, string=text)
                    return text
                elif delete_pods.count(new_str_d) <= 1:
                    print(delete_pods, '删除pods')
                    text = 'Action : `{}`-`{}`\nTime : {}\nMessage : {}'.format(kind, reason, time, msg)
                    worktile_webhook(hookurl=wt_webhook_url, string=text)
                    return text

# 定时清理
import schedule
import time

def job():
    create_pods.clear()
    delete_pods.clear()


app = Flask(__name__)
@app.route('/kubewatch', methods=['GET', 'POST'])
def webhook():
    if request.method == 'GET':
        logging.debug('Get Method not implemented')
        return jsonify({'status': 'success'}), 200

    elif request.method == 'POST':
        schedule.every().hour.do(job)
        payload = request.json
        sim = Json_sik().js(payload=payload)
        return jsonify({'status': 'success'}), 200
    else:
        abort(400)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port='10010')

posted @ 2021-05-29 18:31 SRE运维充电站阅读(810) 评论(0) 编辑收藏举报

刷新页面返回顶部

SRE运维充电站

Monitoring-K8s资源观测工具Kubewatch

一、Kubewatch介绍

二、Helm部署kubewatch

三、Webhook 配置例子

公告