Python异常检测以及报警实现

# -*- encoding: utf-8 -*-

'''

@File    :   launcher.py

@Time    :   2021/11/16 17:55:00

@Author  :   Shydow

@Version :   1.0

@Desc    :   3sigma异常值检测

'''

# here put the import lib

import numpy as np

import pandas as pd

from matplotlib import pyplot as plt

import seaborn as sns

import requests

from DingDingBot import DDBOT

def load_data(cols:list, filter: str):

    """ 加载数据 """

    df = pd.read_csv("data\detail.csv", encoding="utf-8")

    p_col = ["stat_day", "stat_hours", "app_channel", "pv", "ids_uv", "users_uv"]

    df.columns = p_col

    df = df[df["app_channel"] == filter]

    return df[cols]

def sigma_stat(N: int):

    """ N is k sigma """

    # 预处理数据

    cols = ["stat_day", "stat_hours", "pv"]

    init_data = load_data(cols, "Shanghai")

    init_data["stat_hours"] = init_data["stat_hours"].map(str)

    init_data["date"] = init_data["stat_day"].str.cat(init_data["stat_hours"], sep=" ")

   

    # 模型初始化

    data_y = init_data["pv"]

    data_x = init_data["date"]

    ymean = np.mean(data_y)

    ystd = np.std(data_y)

    threshold1 = ymean - N * ystd

    threshold2 = ymean + N * ystd

    print("pv平均值：{0}, 标准差：{1}, sigma-min：{2}, sigma-max：{3}".format(ymean, ystd, threshold1, threshold2))

    # 将异常值保存

    outlier = []

    outlier_x = []

    for i in range(0, len(data_y)):

        if (data_y.iloc[i] < threshold1 or data_y.iloc[i] > threshold2):

            outlier.append(data_y.iloc[i])

            outlier_x.append(data_x.iloc[i])

        else:

            continue

   

    result = zip(outlier_x, outlier)

    print("\n异常数据如下：\n")

    for item in result:

        print(item)

    return data_y

def quantile_stat(data_x, data_y):

    """ 分位数异常检测 """

    # sns.boxplot(data_y) 箱线图

    # 计算上四分位数和下四分位数

    q1 = data_y.quantile(0.25)

    q3 = data_y.quantile(0.75)

    low_outlier = q1 - 1.5 * (q3 - q1)

    high_outlier = q3 + 1.5 * (q3 - q1)

    # 检测异常点

    q_outlier = []

    q_outlier_x = []

    for i in range(0, len(data_y)):

        if (data_y.iloc[i] < lower_outlier or data_y.iloc[i] > high_outlier):

            q_outlier.append(data_y.iloc[i])

            q_outlier_x.append(data_x.iloc[i])

        else:

            continue

def send_msg():

    """ 钉钉异常检测报警 """

    pass

   

if __name__ == '__main__':

    pass