Prometheus使用企业微信Webhook推送告警通知
构建微信Webhook镜像
代码依赖文件：requirements.txt
certifi==2018.10.15
chardet==3.0.4
Click==7.0
Flask==1.0.2
idna==2.7
itsdangerous==1.1.0
Jinja2==2.10
MarkupSafe==1.1.0
requests==2.20.1
urllib3==1.24.1
Werkzeug==0.15.3
微信Webhook代码示例文件：app.py
import os
import sys
import json
import logging
import requests
from urllib.parse import urlparse
from flask import Flask
from flask import request
import urllib3
urllib3.disable_warnings()
app = Flask(__name__)
logging.basicConfig(
level=logging.DEBUG if os.getenv('LOG_LEVEL') == "debug" else logging.INFO,
format="%(asctime)s %(levelname)s %(message)s")
@app.route('/', methods=['POST', 'GET'])
def send():
if request.method == "POST":
post_data = request.get_data()
app.logger.debug(post_data)
alert_data = json.loads(post_data)
send_message(alert_data)
return "Success!"
else:
return "Weclome to use weixin webhook server!"
def send_message(alert_data):
corp_id = os.getenv('CORP_ID')
secret = os.getenv('SECRET')
token = get_token(corp_id, secret)
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % token
user_id = os.getenv('USER_ID')
party_id = os.getenv('PARTY_ID')
agent_id = os.getenv('AGENT_ID')
alerts = alert_data['alerts']
alert_name = alerts[0]['labels']['alertname']
subject = "[容器告警通知] %s 有 %d 条新的报警" % (alert_name, len(alerts))
external_url = alerts[0]['generatorURL']
prometheus_url = os.getenv('PROME_URL')
if prometheus_url:
res = urlparse(external_url)
external_url = external_url.replace(res.netloc, prometheus_url)
data = {
"touser": user_id,
#"totag": tag_id,
"toparty": party_id,
"msgtype": "markdown",
"agentid": agent_id,
"markdown": {
"content": subject + "\n" + _mark_item(alerts[0]) + "\n" + "[>>点击查看完整信息](" + external_url + ")\n"
},
"safe": "0"
}
r = requests.post(url=url,data=json.dumps(data),verify=False)
n = 0
while r.json()['errcode'] != 0 and n < 4:
n+=1
token = get_token(corp_id, secret)
if token:
url = "https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token=%s" % token
r = requests.post(url=url,data=json.dumps(data),verify=False)
return r.json()
def get_token(corp_id,secret):
url = "https://qyapi.weixin.qq.com/cgi-bin/gettoken"
data = {
"corpid":corp_id,
"corpsecret":secret
}
r = requests.get(url=url,params=data,verify=False)
print(r.json())
if r.json()['errcode'] != 0:
return False
else:
token = r.json()['access_token']
return token
def _mark_item(alert):
labels = alert['labels']
annotations = ""
for k, v in alert['annotations'].items():
annotations += "{0}: {1}\n".format(k, v)
if "job" in labels:
mark_item = "\n> job: " + labels['job'] + "\n\n" + annotations + "\n"
else:
mark_item = "\n> " + annotations + "\n"
return mark_item
if __name__ == '__main__':
app.debug=False
app.run(host='0.0.0.0', port=5000)
镜像构建模板文件：Dockerfile
FROM python:3.6.4-alpine3.4
MAINTAINER varden
RUN echo "https://mirrors.aliyun.com/alpine/v3.4/main/" > /etc/apk/repositories
RUN echo "https://mirrors.aliyun.com/alpine/v3.4/community/" >> /etc/apk/repositories
RUN apk update
RUN apk upgrade
RUN apk add --no-cache ca-certificates tzdata curl bash && rm -rf /var/cache/apk/*
RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN pip install -r requirements.txt -i http://mirrors.aliyun.com/pypi/simple/ --trusted-host mirrors.aliyun.com
COPY app.py /app/app.py
CMD python app.py
构建命令
docker build -t weixin-hook:1.0.0 .
docker部署
#!/bin/bash
#
set -u
docker run -detach \
--publish 5000:5000 \
--name weixin-hook \
--restart always \
--env LOG_LEVEL=debug \
--env PROME_URL=10.10.10.200:32101 \
--env CORP_ID=xxxxxxxxxxxxxxx \
--env SECRET=xxxxxxxxx-xxxxxxxxxxxxxxxxxxxxx \
--env USER_ID="" \
--env PARTY_ID=2 \
--env AGENT_ID=1000002 \
weixin-hook:1.0.0
docker-compose部署
version: "3.3"
networks:
bridge_net:
external:
name: bridge
services:
weixin_hook:
image: weixin-hook:1.0.0
ports:
- "15000:5000"
networks:
- bridge_net
environment:
LOG_LEVEL: debug
PROME_URL: 10.10.10.200:32101
env_file: robot.env
deploy:
mode: replicated
replicas: 1
resources:
limits:
cpus: '0.2'
memory: 128M
reservations:
cpus: '0.1'
memory: 64M
healthcheck:
test: curl -f http://localhost:5000 || exit 1
interval: 30s
timeout: 30s
retries: 5
环境变量文件：robot.env
CORP_ID=xxxxxxxxxxxxxxx
SECRET=xxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxx
USER_ID=""
PARTY_ID=2
AGENT_ID=1000002
K8s部署
创建Secret资源对象
kubectl create secret generic weixin-secret \
--from-literal=corp_id=xxxxxxxxxxxxxxxx \
--from-literal=secret=xxxxxxxx-xxxxxxxxxxxxxx \
--from-literal=user_id="" \
--from-literal=party_id=2 \
--from-literal=agent_id=1000002 \
-n monitoring
squid代理访问外网：
kubectl create secret generic proxy-secret \
--from-literal=http_proxy=http://<username>:<password>@10.10.10.15:3128 \
--from-literal=https_proxy=http://<username>:<password>@10.10.10.15:3128 \
-n monitoring
K8s部署清单
apiVersion: apps/v1
kind: Deployment
metadata:
name: weixin-hook
namespace: monitoring
spec:
replicas: 1
selector:
matchLabels:
app: weixin-hook
template:
metadata:
labels:
app: weixin-hook
spec:
containers:
- name: weixin-hook
image: weixin-hook:1.0.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5000
name: http
env:
- name: PROME_URL
value: 10.10.10.200:32101
- name: LOG_LEVEL
value: debug
- name: CORP_ID
valueFrom:
secretKeyRef:
name: weixin-secret
key: corp_id
- name: SECRET
valueFrom:
secretKeyRef:
name: weixin-secret
key: secret
- name: USER_ID
valueFrom:
secretKeyRef:
name: weixin-secret
key: user_id
- name: PARTY_ID
valueFrom:
secretKeyRef:
name: weixin-secret
key: party_id
- name: AGENT_ID
valueFrom:
secretKeyRef:
name: weixin-secret
key: agent_id
- name: http_proxy
valueFrom:
secretKeyRef:
name: proxy-secret
key: http_proxy
- name: https_proxy
valueFrom:
secretKeyRef:
name: proxy-secret
key: https_proxy
resources:
requests:
cpu: 100m
memory: 200Mi
limits:
cpu: 100m
memory: 200Mi
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 5000
initialDelaySeconds: 30
timeoutSeconds: 30
readinessProbe:
httpGet:
scheme: HTTP
path: /
port: 5000
initialDelaySeconds: 30
timeoutSeconds: 30
---
apiVersion: v1
kind: Service
metadata:
name: weixin-hook
namespace: monitoring
spec:
selector:
app: weixin-hook
ports:
- name: hook
port: 5000
targetPort: http
在AlertManager中webhook地址直接通过DNS形式访问即可：
receivers:
- name: 'webhook'
webhook_configs:
- url: 'http://weixin-hook:5000'
send_resolved: true
