利用定时任务监控k8s集群容器状态和容器日志中的error并通告警

背景介绍

利用定时任务监控k8s集群容器状态和容器日志中的error并通告警，可以通过企业微信告警、通过邮件告警通知。

脚本说明

#!/bin/bash
# Descriptions: 查询 Pod 日志从 10 分钟之前开始的日志，是否包含 error 或者 exception 关键字

PATH=$PATH:/usr/local/bin

# 脚本执行时间参数，取当前时间的十分钟之前，由于时区关系，时间需要减 8 小时
QueryTime=$(date +"%Y-%m-%dT%H:%M:%S" -d '-8 hours -10 minutes').0000000Z


# 企业通知显示时间
startTime=$(date +"%Y-%m-%dT%H:%M:%S" -d '-10 minutes')

# 企业通知显示截止时间
endTime=$(date +"%Y-%m-%dT%H:%M:%S")

# 需要查询日志的服务清单
DeploymentNames=(
  maorong-bank-attachment \
  maorong-bank-standard \
  maorong-auth \
  maorong-file \
  maorong-gateway \
  maorong-system \
  vchain-afterloan \
  vchain-auth \
  vchain-credit \
  vchain-finance \
  vchain-gateway \
  vchain-iot \
  vchain-iot-sdk-client \
  vchain-living \
  vchain-operating \
  vchain-sic-third \
  vchain-base \
  vchain-inloan \
  vchain-preloan \
  vchain-sic-cfca \
  vchain-sic-msg \
  vchain-sic-blockchain
)

# 企业微信机器人URL
logRobotUrl="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=ebfdaf5094f3f0"
livenssRobotUrl="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=2ab65b7d-7898ad8"

# 服务所在的命名空间
Namespace="haian-prod"

# Pod 日志查询记录文件存放位置
pods_log_file="/var/log/checklogs/pods_log.txt"

# 日志详细信息压缩文件
logs_zipfile=/tmp/logs-$(data +"%Y-%m-%dT%H%M%S").zip
pods_log_dir="/var/log/checklogs/logs"

# deployment 服务状态
deployments_status_file="/var/log/checklogs/deployments_status.txt"

# 服务存活状态
livenessState_file="/var/log/checklogs/liveness_state.txt"

# 邮件收件人地址
mailToUser="
  changli.tang@vonechain.com \
  jiaojiao.zou@vonechain.com
"

# 邮件抄送人地址
mailCcUser="bin.zhou@vonechain.com,"

# 服务健康检查接口配置
health_api=(认证中心_vchain-auth_auth \
  基础服务_vchain-base-biz_base \
  贷前服务_vchain-preloan-biz_preloan \
  贷中服务_vchain-inloan-biz_inloan \
  贷后服务_vchain-afterloan-biz_afterloan \
  运营服务_vchain-operating-biz_operating \
  区块链服务_vchain-sic-blockchain-biz_blockchain \
  第三方服务_vchain-sic-third-biz_third \
  物联网服务_vchain-iot-biz_iot \
  短信服务_vchain-sic-msg-biz_msg \
  cfca认证服务_vchain-sic-cfca-biz_cfca \
  物联网sdk服务_vchain-iot-sdk-client-biz_iot-sdk \
  融资核心服务_vchain-finance-biz_finance \
  预授信场景服务_vchain-credit-biz_credit \
  活体质押融资服务_vchain-living-biz_living 
  银行接口服务_bank-standard_bank-standard
  银行接口附件服务_bank-attachment__bank-attachment
)

# vchain gateway 域名配置
INGRESS_HOST="192.168.10.50"
SCHEMA_PORT="80"
GATEWAY_DOMAIN="prodvchain-gateway.ntmaorong.com"
RESOLVE=${GATEWAY_DOMAIN}:${SCHEMA_PORT}:${INGRESS_HOST}
VCHAIN_GATEWAY_SVC_ADDRESS=$(kubectl get svc -n haian-prod -l app=vchain-gateway -o jsonpath='{.items[0].spec.clusterIP}')
VCHAIN_GATEWAY_SVC_PORT=$(kubectl get svc -n haian-prod -l app=vchain-gateway -o jsonpath='{.items[0].spec.ports[?(@.name == "tcp-9999")].port}')

# maorong gateway 服务配置
MAORONG_GATEWAY_SVC_ADDRESS=$(kubectl get svc -n haian-prod -l app=maorong-gateway -o jsonpath='{.items[0].spec.clusterIP}')
MAORONG_GATEWAY_SVC_PORT=$(kubectl get svc -n haian-prod -l app=maorong-gateway -o jsonpath='{.items[0].spec.ports[?(@.name == "tcp-8080")].port}')

# 判断文件目录是否存在
[ ! -d ${pods_log_file%/*} ] && mkdir -p ${pods_log_file%/*}
[ ! -d ${deployments_status_file%/*} ] && mkdir -p ${deployments_status_file%/*}
[ ! -f ${pods_log_file} ] && touch ${pods_log_file}
[ ! -f ${deployments_status_file} ] && touch ${deployments_status_file}
[ ! -f ${livenessState_file} ] && touch ${livenessState_file}

# 查询日志匹配关键字
PATTERN="\[ERROR"
# 忽略提醒关键字
IGNORE_PATTERN="验证码不正确\|Invalid cookie\|验证码过期或不存在\|供应商未网签银行"

# 查询服务日志
function tail_pods_log(){
    echo "日志开始时间为: ${startTime}, 截止时间为: ${endTime}"
    echo ""
    for item in ${DeploymentNames[@]};do
        # 查询 Pods 的名称
        podsName=(`kubectl get pods -n ${Namespace} -l app=${item} -o jsonpath='{range .items[*]}{.metadata.name}{" "}{end}'`)
        if [[ ${#podsName[@]} -gt "1" ]];then
            for pod in ${podsName[@]};do
                echo "正在查询 Pod [${pod}] 的日志..."
                kubectl logs -n ${Namespace} --since-time="$QueryTime" ${pod} | grep -v "$IGNORE_PATTERN" | grep  "${PATTERN}" -A100 -B100 > ${pods_log_dir}/${pod}.log 2>&1
                if [[ $? -eq 0 ]];then
                    echo -e "\033[1;33;41m查询到 error 或者 exception 关键字的日志。查询日志时间范围为 ${startTime} ~ ${endTime}, Pod 名称为: ${pod}, 命名空间为: ${Namespace}.\e[0m"
                    echo "时间范围: [\`${startTime}\` ~ \`${endTime}\`]; pod 名称: [\`${pod}\`];  命名空间: [\`${Namespace}\`]" >> ${pods_log_file}
                else
                    echo "未查询到错误日志."
                fi
            done 
        else
            echo "正在查询 Pod [${podsName[0]}] 的日志..."
            kubectl logs -n ${Namespace} --since-time="$QueryTime" ${podsName} | grep -v "$IGNORE_PATTERN" | grep  "${PATTERN}" -A100 -B100 > ${pods_log_dir}/${podsName}.log 2>&1
            if [[ $? -eq 0 ]];then
                echo -e "\033[1;33;41m查询到 error 或者 exception 关键字的日志。查询日志时间范围为 ${startTime} ~ ${endTime}, Pod 名称为: ${podsName}, 名空间为: ${Namespace}.\e[0m" 
                echo "时间范围: [\`${startTime}\` ~ \`${endTime}\`]; pod 名称: [\`${podsName}\`]; 命名空间: [\`${Namespace}\`]" >> ${pods_log_file}
            else
                echo "未查询到错误日志."
            fi
        fi
    
    done
}


# 查询服务存活状态
function get_liveness_state(){
    echo "查询的命名空间为: [<font color='warning'>${Namespace}</font>]\n>" >> ${livenessState_file}
    for item in ${health_api[@]};do
        serverName=$(echo ${item} | awk  -F '_' '{print $2}')
        apiUrl=${item##*_}
        echo "正在查询服务 [${item%%_*} - ${serverName}] 的状态..."
        if [[ ${apiUrl} != "bank-standard" ]] && [[ ${apiUrl} != "bank-attachment" ]];then
            livenessState=$(curl -s http://${VCHAIN_GATEWAY_SVC_ADDRESS}:${VCHAIN_GATEWAY_SVC_PORT}/${apiUrl}/actuator/health |jq ".components.livenessState.status" | awk -F '"' '{print $2}')
        else
            livenessState=$(curl -s http://${MAORONG_GATEWAY_SVC_ADDRESS}:${MAORONG_GATEWAY_SVC_PORT}/${apiUrl}/actuator/health |jq ".status" | awk -F '"' '{print $2}')
        fi

        if [[ "$livenessState" != "UP" ]];then
            echo -e "\033[1;33;41m服务 [${item%%_*} - ${serverName}] 服务异常，状态为非 UP!!\e[0m"
            echo "serverName: [<font color='warning'>${serverName}</font>]; status: [<font color='comment'>DOWN</font>]; aliasName: [<font color='warning'>${item%%_*}</font>]" >> ${livenessState_file}
            #echo "serverName: [<font color='warning'>${serverName}</font>]; status: [<font color='comment'>${livenessState}</font>]; aliasName: [<font color='warning'>${item%%_*}</font>]" >> ${livenessState_file}
        else
            echo -e "服务 \033[1;33;32m[${item%%_*} - ${serverName}] \e[0m存活状态为 \033[1;33;32m${livenessState}.\e[0m"
            echo "serverName: [<font color='warning'>${serverName}</font>]; status: [<font color='info'>${livenessState}</font>]; aliasName: [<font color='warning'>${item%%_*}</font>]" >> ${livenessState_file}
        fi
    done
}


# 获取 deployment 副本控制器
function get_deployment_status(){
    for item in ${DeploymentNames[@]};do
        echo "正在查询服务 [${item}] 的状态..."
        availableReplicas=$(kubectl get deployment -n ${Namespace} -l app=${item} -o jsonpath='{.items[0].status.availableReplicas}')
        replicas=$(kubectl get deployment -n ${Namespace} -l app=${item} -o jsonpath='{.items[0].status.replicas}')
        if [[ $availableReplicas != $replicas  ]];then
            echo -e "\033[1;33;41m${item}服务状态异常，可用副本数不等于目标副本数！\e[0m"
            echo "服务名: ${item}; 可用副本数: ${availableReplicas}; 目标副本数: ${replicas}" >> ${deployments_status_file}
        else
            echo -e "\033[1;33;32m${item} \e[0m服务状态正常. 可用副本数: ${availableReplicas}. 目标副本数: ${replicas}"
        fi
     done
}

# 发送消息到用户邮箱
send_email(){
    echo -e "生产环境产生错误日志，相关信息如下: \
        \n`cat ${2}`
        \n查询日志时使用命令如下: \
        \nkubectl logs -n haian-prod --timestamps --since-time=${3} \${PodName}" \
        | mail -s "${1}" -a ${2} -c ${4} ${5}
}

# 发送 checklogs 脚本执行内容到企业微信机器人
function send_pods_log_messages(){
    # 生成日志查询记录文件
    tail_pods_log

    data=$(cat ${pods_log_file})

    # 删除内容为空的日志文件
    for file in `ls ${pods_log_dir}`;do
        if [[ ! -s $file ]];then
            rm -rf ${pods_log_dir}/$file
        fi
    done
    # 压缩日志文件
    if [[ "$(ls -A ${pods_log_dir})" ]];then
        zip -r ${logs_zipfile} ${pods_log_dir}
    
        if [[ $? -eq 0 ]];then
            echo "文件压缩完成"
        else
            echo "文件压缩失败"
        fi
    else
        echo "${pods_log_dir} 目录为空"
    fi
    if [[ $data != "" ]];then
        # 发送消息到企业微信机器人
        curl ${logRobotUrl} -H 'Content-Type: application/json' -d '{
          "msgtype": "markdown",
          "markdown": {
            "content": "<font color='warning'>发现错误日志。相关信息为:</font>
                        > '"$data"' \n>
                        > <font color='warning'>**查询日志时使用命令如下:** </font>
                        > `kubectl logs -n '"${Namespace}"' --timestamps --since-time=\"'"${QueryTime}"'\" ${PodName} `\n>
                        > <font color='info'>将以上命令中的 PodName 替换为实际的 Pod 名称即可</font>"
          }
        }'
        # 发送邮件
        if [[ -e ${logs_zipfile} ]];then
            echo -e "生产环境产生错误日志，相关信息如下: \
              \n`cat ${pods_log_file}` \
              \n查询日志时使用命令如下: \
              \nkubectl logs -n haian-prod --timestamps --since-time=${QueryTime} \${PodName}" \
              | mail -s "项目-生产环境-错误日志" -a ${logs_zipfile} -c ${mailCcUser} ${mailToUser}
        else
             echo -e "生产环境产生错误日志，相关信息如下: \
              \n`cat ${pods_log_file}` \
              \n查询日志时使用命令如下: \
              \nkubectl logs -n haian-prod --timestamps --since-time=${QueryTime} \${PodName}" \
              | mail -s "项目-生产环境-错误日志" -c ${mailCcUser} ${mailToUser}
        fi
	# 清空日志文件
        echo "" > ${pods_log_file}
        rm -rf ${logs_zipfile}
    else
        echo "不存在错误日志"
    fi
}


# 发送 deployment 副本状态到企业微信机器人
function send_deployments_status_messages(){
    # 生成 deployments 状态记录文件
    get_deployment_status

    data=$(cat ${deployments_status_file})
    if [[ $data != "" ]];then
        curl ${logRobotUrl} -H 'Content-Type: application/json' -d '{
          "msgtype": "markdown",
          "markdown": {
            "content": "<font color='warning'>当前 deployment 服务状态信息如下:</font>
                       > '"$data"' \n>
                       > <font color='warning'>**查询deployment 状态使用命令如下:** </font>
                       > `kubectl get deployment -n ${Namespace} ${DeloymentName}` \n>
                       > <font color='info'>将以上命令中的变量替换为实际的值即可</font>"
          }
        }'
	# 发送邮件
        #echo -e "项目生产环境 Deployment 状态信息如下: \
        #  \n`cat ${deployments_status_file}`" \
        #  | mail -s "项目-生产环境-Deployment 状态" -a ${deployments_status_file} -c ${mailCcUser} ${mailToUser}
        # 清空文件       
        echo "" > ${deployments_status_file}
    else
        echo -e "\033[1;33;32m所有服务状态正常.\e[0m"
    fi
}


# 发送服务存活状态到企业微信机器人
function send_liveness_stats_messages(){
    # 生成服务存活数据
    get_liveness_state

    data=$(cat ${livenessState_file})
    curl ${livenssRobotUrl} -H 'Content-Type: application/json' -d '{
      "msgtype": "markdown",
       "markdown": {
         "content": "<font color='warning'>当前服务存活状态信息为:</font>
                    > '"$data"' \n>"
      }
    }'
    # 发送邮件
    #echo -e "项目生产环境服务存活状态信息如下: \
    #  \n`cat ${livenessState_file}`" \
    #  | mail -s "项目-生产环境-服务状态" -a ${livenessState_file} -c ${mailCcUser} ${mailToUser}
    # 清空文件
    echo "" > ${livenessState_file}
    
}


# 脚本使用说明
function Usage(){
    echo "$0 可用参数说明如下:"
    echo "  -l: 查询 Pods 日志"
    echo "  -d: 查询 Deployment 副本状态"
    echo "  -s: 查询服务存活状态"
    echo "  -h: 帮助信息"

}

while getopts "ldsh?*" arg
do
    case $arg in
        l)
            echo "查询 pods 日志"
            send_pods_log_messages
            ;;
        d)
            echo "查询 Deployment 副本状态"
            send_deployments_status_messages
            ;;
        s)
            echo "查询服务的存活状态"
            send_liveness_stats_messages
            ;;
        h)
            Usage
            ;;
        *)
            echo ”参数错误“
            Usage
            ;;
    esac
done

发邮件

#!/bin/bash
# 发送信息邮件

QueryTime=$(date +"%Y-%m-%dT%H:%M:%S" -d '-8 hours -10 minutes').0000000Z
pods_log_file="test.txt"

# 邮件收件人
EMAIL_RECIVER="bin.zhou@vonechain.com"

# 邮件抄送人
#EMAIL_CC="jiaojiao.zou@vonechain.com,"

# 邮件正文
EMAIL_CONTENT="./test.txt"

#send_email(){
#    echo -e "生产环境产生错误日志，相关信息如下: \
#        \n`cat ${2}`
#        \n查询日志时使用命令如下: \
#        \nkubectl logs -n haian-prod --timestamps --since-time=${3} \${PodName}" \
#        | mail -s "${1}" -a ${4} -c ${5} ${6}
#}

send_email(){
  echo -e $2 | mail -s $1 -a ${4} -c ${5} ${6}
}

subject="项目-生产环境-产生错误日志"
echo -e "生产环境产生错误日志，相关信息如下: \
  \n`cat ${pods_log_file}` \
  \n查询日志时使用命令如下: \
  \nkubectl logs -n haian-prod --timestamps --since-time='${QueryTime}' \${PodName}
  \n
" | mail -s ${subject} -a ${pods_log_file} ${EMAIL_RECIVER}
#send_email ${subject} ${pods_log_file} ${QueryTime} ${pods_log_file} ${EMAIL_CC} ${EMAIL_RECIVER}
#echo -e ${content} | mail -s ${subject} -a ${pods_log_file} ${EMAIL_RECIVER}

配置定时任务

0 */2 * * * /root/scripts/checklogs.sh -d >/tmp/crontab_deployment_status.log 2>&1
0 */2 * * * /root/scripts/checklogs.sh -s >/tmp/crontab_liveness_status.log 2>&1
*/10 * * * * /root/scripts/checklogs.sh -l >/tmp/crontab_checklogs.log 2>&1

效果