nginx日志告警

k8s的ingress日志用fluent bit收集写入es,然后读取es里面nginx日志，匹配关键条件触发告警，例如匹配nginx日志里面path包含xxx，触发告警：

import requests,json,pytz
from requests.auth import HTTPBasicAuth
import time,hashlib,hmac,urllib,base64,datetime
from apscheduler.schedulers.blocking import BlockingScheduler

def get_code():
    webhook = get_webhook(1)
    url = 'http://elasticsearch-logs:9200/_xpack/sql'
    headers = {'Content-Type': 'application/json'}
    login = HTTPBasicAuth('elastic', 'xxxxx')
    query = {
        'query': '''
            select request_query,path,count(1)cnt from \"ingress-nginx\" ma 
                    where (ma.path like  '/api/v3/auth/action/code%' or ma.path like  '/api/v2/auth/smscode%' or ma.path like  '/api/v2/auth/mailcode%' 
                                      or ma.path like '/v2/wallet/fcoin/order%' or ma.path like '/api/v4/social/blog/reward%' ) 
                    and \"@timestamp\" >=current_timestamp -interval 2 minute group by 1,2 having count(1) >=4
       '''
    }
    res = requests.post(url, headers=headers, auth=login, data=json.dumps(query))
    for row in res.json()['rows']:
        message = {"url": row[0], "path": row[1], "num": row[2]}
        if message:
            url = str(message['url'])
            print(url)
            path = str(message['path'])
            cnt = str(message['num'])
            headers = {'Content-Type': 'application/json'}
            data = {
                "msgtype": "markdown",
                "markdown": {
                    "title": "NGINX log alert" + "....",
                    "text": "<font color=#FF0000 size=3>最近3min频繁请求:</font> "
                            + "\n\n>请求参数 ：" + url
                            + "\n\n>参数路径：" + path +
                            "\n\n>请求次数：" + cnt
                },
                "at": {
                    "atMobiles": [123456],
                    "isAtAll": False
                }
            }
        requests.post(url=webhook, data=json.dumps(data), headers=headers)
        print(data)


URL="https://oapi.dingtalk.com/robot/send?access_token=881781b6330adf86bd520f7a7c33dad05c619962454xxxx"
secret = "xxxx"


def get_timestamp_sign():
    timestamp = str(round(time.time() * 1000))
    secret_enc = secret.encode('utf-8')
    string_to_sign = '{}\n{}'.format(timestamp, secret)
    string_to_sign_enc = string_to_sign.encode('utf-8')
    hmac_code = hmac.new(secret_enc, string_to_sign_enc,
                         digestmod=hashlib.sha256).digest()
    sign = urllib.parse.quote_plus(base64.b64encode(hmac_code))
    return (timestamp, sign)

def get_signed_url():
    timestamp, sign = get_timestamp_sign()
    webhook = URL + "&timestamp="+timestamp+"&sign="+sign
    return webhook

def get_webhook(mode):
    if mode == 0: # only 敏感字
       webhook = URL
    elif mode == 1 or  mode ==2 : # 敏感字和加签 或 # 敏感字+加签+ip
        webhook = get_signed_url()
    else:
        webhook = ""
        print("error! mode:   ",mode,"  webhook :  ",webhook)
    return webhook

def fun_code():
    now = datetime.datetime.now()
    ts = now.strftime('%Y-%m-%d %H:%M:%S')
    get_code()
    print('do fun_code  time :',ts)


def tasklist():
    #创建调度器：BlockingScheduler
    scheduler = BlockingScheduler()
    #添加任务,时间间隔60S
    scheduler.add_job(fun_code, 'interval', seconds=60, id='job1',timezone=pytz.timezone('Asia/Shanghai'))
    scheduler.start()
tasklist()

这样简单一个nginx日志告警就完成了，做成pod运行于k8s就行了。