AlertManager简介与告警模板
目录
AlertManager
altermanager是普罗米修斯的报警组件，主要是用针对异常数据进行报警。首先创建一个报警的规则，其次创建路由（给谁发报警信息）。
部署
alertmanager部署有两种方式
1、容器化部署
2、安装包部署
1、容器化部署
1.1、部署Alertmanager
[root@k8s-master-01 ~]# cat /root/kube-prometheus/manifests/alertmanager-alertmanager.yaml
apiVersion: monitoring.coreos.com/v1
kind: Alertmanager
metadata:
labels:
alertmanager: main
name: main
namespace: monitoring
spec:
image: quay.io/prometheus/alertmanager:v0.20.0
nodeSelector:
kubernetes.io/os: linux
#默认部署3个
replicas: 3
#资源可用大小
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 1000
#权限
serviceAccountName: alertmanager-main
#版本
version: v0.20.0
- 查看部署结果
[root@k8s-master-01 alertmanager]# kubectl apply -f alertmanager.yaml
alertmanager.monitoring.coreos.com/main created
[root@k8s-master-01 alertmanager]# kubectl get pod -n monitoring
NAME READY STATUS RESTARTS AGE
alertmanager-main-0 2/2 Running 0 9m38s
alertmanager-main-1 2/2 Running 0 9m38s
alertmanager-main-2 2/2 Running 0 9m38s
1.2、增加一个ingress暴漏服务
kind: Ingress
apiVersion: extensions/v1beta1
metadata:
name: alertmanager
namespace: monitoring
spec:
rules:
- host: "www.altermanager.cluster.local.com"
http:
paths:
- backend:
serviceName: alertmanager-main
servicePort: 9093
path: /
- 查看部署结果
[root@k8s-master-01 alertmanager]# kubectl apply -f ingress.yaml
Warning: extensions/v1beta1 Ingress is deprecated in v1.14+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
ingress.extensions/alertmanager created
[root@k8s-master-01 alertmanager]# kubectl get ingress -n monitoring
NAME CLASS HOSTS ADDRESS PORTS AGE
alertmanager <none> www.altermanager.cluster.local.com 192.168.15.32 80 18s
1.3、浏览器访问
[root@k8s-master-01 alertmanager]# kubectl get ingress -n monitoring
NAME CLASS HOSTS ADDRESS PORTS AGE
alertmanager <none> www.altermanager.cluster.local.com 192.168.15.32 80 72m
[root@k8s-master-01 alertmanager]# kubectl get svc -n ingress-nginx
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
ingress-nginx-controller NodePort 10.100.56.109 <none> 80:32755/TCP,443:30229/TCP 26d
2、安装包部署
[root@tomcat]# wget https://github.com/prometheus/alertmanager/releases/download/v0.21.0/alertmanager-0.21.0.linux-amd64.tar.gz
[root@tomcat]# tar -xf alertmanager-0.21.0.linux-amd64.tar.gz -C /usr/local/
[root@tomcat]# vim /etc/profile
export ALTERMANAGER=/usr/local/alertmanager-0.21.0.linux-amd64
PATH=$PATH:$GO_HOME:$GO_ROOT:$GO_PATH:$GO_HOME/bin:$PROMETHEUS_HOME:$ALTERMANAGER
[root@tomcat ~]# /usr/local/alertmanager-0.21.0.linux-amd64/alertmanager --config.file=/usr/local/alertmanager-0.21.0.linux-amd64/alertmanager.yml
3、报警
3.1、创建报警规则
[root@k8s-master-01 alertmanager]# cat alertmanager.yaml
global:
resolve_timeout: 1h # 警告的间隔时间，默认是：5m
# 邮件告警配置(发送邮件的邮箱信息)
smtp_smarthost: 'smtp.qq.com:465'
smtp_from: 'axxxy@qq.com'
smtp_auth_username: 'axxxy@qq.com'
smtp_auth_password: 'qxxxxb'
smtp_require_tls: false
# 配置发送的报警信息的模板
templates:
- '/etc/alertmanager/config/*.tmpl'
# 路由
route:
# 匹配的标签的ksy
group_by: ['severity']
group_wait: 30s
group_interval: 5m
repeat_interval: 4h
receiver: 'email'
routes:
- receiver: 'email'
match_re:
# 匹配的是标签的值
service: '^(warning|critical)$'
# 报警信息的接收者
receivers:
- name: 'email'
email_configs:
- to: '12xxxx30@qq.com'
3.2、添加报警
[root@kubernetes-master-01 altermanager]# kubectl create secret generic -n monitoring alertmanager-main --from-file=alertmanager.yaml --from-file=email.tmpl --dry-run -oyaml | kubectl replace -f -
4、模板
4.1、alertmanager
# global块配置下的配置选项在本配置文件内的所有配置项下可见
global:
# 在Alertmanager内管理的每一条告警均有两种状态: "resolved"或者"firing". 在altermanager首次发送告警通知后, 该告警会一直处于firing状态,设置resolve_timeout可以指定处于firing状态的告警间隔多长时间会被设置为resolved状态, 在设置为resolved状态的告警后,altermanager不会再发送firing的告警通知.
resolve_timeout: 1h
# 邮件告警配置
smtp_smarthost: 'smtp.exmail.qq.com:25'
smtp_from: 'dukuan@xxx.com'
smtp_auth_username: 'dukuan@xxx.com'
smtp_auth_password: 'DKxxx'
# HipChat告警配置
# hipchat_auth_token: '123456789'
# hipchat_auth_url: 'https://hipchat.foobar.org/'
# wechat
wechat_api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
wechat_api_secret: 'JJ'
wechat_api_corp_id: 'ww'
# 告警通知模板
templates:
- '/etc/alertmanager/config/*.tmpl'
# route: 根路由,该模块用于该根路由下的节点及子路由routes的定义. 子树节点如果不对相关配置进行配置，则默认会从父路由树继承该配置选项。每一条告警都要进入route，即要求配置选项group_by的值能够匹配到每一条告警的至少一个labelkey(即通过POST请求向altermanager服务接口所发送告警的labels项所携带的<labelname>)，告警进入到route后，将会根据子路由routes节点中的配置项match_re或者match来确定能进入该子路由节点的告警(由在match_re或者match下配置的labelkey: labelvalue是否为告警labels的子集决定，是的话则会进入该子路由节点，否则不能接收进入该子路由节点).
route:
# 例如所有labelkey:labelvalue含cluster=A及altertname=LatencyHigh labelkey的告警都会被归入单一组中
group_by: ['job', 'altername', 'cluster', 'service','severity']
# 若一组新的告警产生，则会等group_wait后再发送通知，该功能主要用于当告警在很短时间内接连产生时，在group_wait内合并为单一的告警后再发送
group_wait: 30s
# 再次告警时间间隔
group_interval: 5m
# 如果一条告警通知已成功发送，且在间隔repeat_interval后，该告警仍然未被设置为resolved，则会再次发送该告警通知
repeat_interval: 12h
# 默认告警通知接收者，凡未被匹配进入各子路由节点的告警均被发送到此接收者
receiver: 'wechat'
# 上述route的配置会被传递给子路由节点，子路由节点进行重新配置才会被覆盖
# 子路由树
routes:
# 该配置选项使用正则表达式来匹配告警的labels，以确定能否进入该子路由树
# match_re和match均用于匹配labelkey为service,labelvalue分别为指定值的告警，被匹配到的告警会将通知发送到对应的receiver
- match_re:
service: ^(foo1|foo2|baz)$
receiver: 'wechat'
# 在带有service标签的告警同时有severity标签时，他可以有自己的子路由，同时具有severity != critical的告警则被发送给接收者team-ops-mails,对severity == critical的告警则被发送到对应的接收者即team-ops-pager
routes:
- match:
severity: critical
receiver: 'wechat'
# 比如关于数据库服务的告警，如果子路由没有匹配到相应的owner标签，则都默认由team-DB-pager接收
- match:
service: database
receiver: 'wechat'
# 我们也可以先根据标签service:database将数据库服务告警过滤出来，然后进一步将所有同时带labelkey为database
- match:
severity: critical
receiver: 'wechat'
# 抑制规则，当出现critical告警时 忽略warning
inhibit_rules:
- source_match:
severity: 'critical'
target_match:
severity: 'warning'
# Apply inhibition if the alertname is the same.
# equal: ['alertname', 'cluster', 'service']
#
# 收件人配置
receivers:
- name: 'team-ops-mails'
email_configs:
- to: 'dukuan@xxx.com'
- name: 'wechat'
wechat_configs:
- send_resolved: true
corp_id: 'ww'
api_secret: 'JJ'
to_tag: '1'
agent_id: '1000002'
api_url: 'https://qyapi.weixin.qq.com/cgi-bin/'
message: '{{ template "wechat.default.message" . }}'
#- name: 'team-X-pager'
# email_configs:
# - to: 'team-X+alerts-critical@example.org'
# pagerduty_configs:
# - service_key: <team-X-key>
#
#- name: 'team-Y-mails'
# email_configs:
# - to: 'team-Y+alerts@example.org'
#
#- name: 'team-Y-pager'
# pagerduty_configs:
# - service_key: <team-Y-key>
#
#- name: 'team-DB-pager'
# pagerduty_configs:
# - service_key: <team-DB-key>
#
#- name: 'team-X-hipchat'
# hipchat_configs:
# - auth_token: <auth_token>
# room_id: 85
# message_format: html
# notify: true
4.2、报警邮件模板
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta name="viewport" content="width=device-width" />
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>{{ template "__subject" . }}</title>
<style>
/* -------------------------------------
GLOBAL
A very basic CSS reset
------------------------------------- */
* {
margin: 0;
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
box-sizing: border-box;
font-size: 14px;
}
img {
max-width: 100%;
}
body {
-webkit-font-smoothing: antialiased;
-webkit-text-size-adjust: none;
width: 100% !important;
height: 100%;
line-height: 1.6em;
/* 1.6em * 14px = 22.4px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
/*line-height: 22px;*/
}
/* Let's make sure all tables have defaults */
table td {
vertical-align: top;
}
/* -------------------------------------
BODY & CONTAINER
------------------------------------- */
body {
background-color: #f6f6f6;
}
.body-wrap {
background-color: #f6f6f6;
width: 100%;
}
.container {
display: block !important;
max-width: 600px !important;
margin: 0 auto !important;
/* makes it centered */
clear: both !important;
}
.content {
max-width: 600px;
margin: 0 auto;
display: block;
padding: 20px;
}
/* -------------------------------------
HEADER, FOOTER, MAIN
------------------------------------- */
.main {
background-color: #fff;
border: 1px solid #e9e9e9;
border-radius: 3px;
}
.content-wrap {
padding: 30px;
}
.content-block {
padding: 0 0 20px;
}
.header {
width: 100%;
margin-bottom: 20px;
}
.footer {
width: 100%;
clear: both;
color: #999;
padding: 20px;
}
.footer p, .footer a, .footer td {
color: #999;
font-size: 12px;
}
/* -------------------------------------
TYPOGRAPHY
------------------------------------- */
h1, h2, h3 {
font-family: "Helvetica Neue", Helvetica, Arial, "Lucida Grande", sans-serif;
color: #000;
margin: 40px 0 0;
line-height: 1.2em;
font-weight: 400;
}
h1 {
font-size: 32px;
font-weight: 500;
/* 1.2em * 32px = 38.4px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
/*line-height: 38px;*/
}
h2 {
font-size: 24px;
/* 1.2em * 24px = 28.8px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
/*line-height: 29px;*/
}
h3 {
font-size: 18px;
/* 1.2em * 18px = 21.6px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
/*line-height: 22px;*/
}
h4 {
font-size: 14px;
font-weight: 600;
}
p, ul, ol {
margin-bottom: 10px;
font-weight: normal;
}
p li, ul li, ol li {
margin-left: 5px;
list-style-position: inside;
}
/* -------------------------------------
LINKS & BUTTONS
------------------------------------- */
a {
color: #348eda;
text-decoration: underline;
}
.btn-primary {
text-decoration: none;
color: #FFF;
background-color: #348eda;
border: solid #348eda;
border-width: 10px 20px;
line-height: 2em;
/* 2em * 14px = 28px, use px to get airier line-height also in Thunderbird, and Yahoo!, Outlook.com, AOL webmail clients */
/*line-height: 28px;*/
font-weight: bold;
text-align: center;
cursor: pointer;
display: inline-block;
border-radius: 5px;
text-transform: capitalize;
}
/* -------------------------------------
OTHER STYLES THAT MIGHT BE USEFUL
------------------------------------- */
.last {
margin-bottom: 0;
}
.first {
margin-top: 0;
}
.aligncenter {
text-align: center;
}
.alignright {
text-align: right;
}
.alignleft {
text-align: left;
}
.clear {
clear: both;
}
/* -------------------------------------
ALERTS
Change the class depending on warning email, good email or bad email
------------------------------------- */
.alert {
font-size: 16px;
color: #fff;
font-weight: 500;
padding: 20px;
text-align: center;
border-radius: 3px 3px 0 0;
}
.alert a {
color: #fff;
text-decoration: none;
font-weight: 500;
font-size: 16px;
}
.alert.alert-warning {
background-color: #E6522C;
}
.alert.alert-bad {
background-color: #D0021B;
}
.alert.alert-good {
background-color: #68B90F;
}
/* -------------------------------------
INVOICE
Styles for the billing table
------------------------------------- */
.invoice {
margin: 40px auto;
text-align: left;
width: 80%;
}
.invoice td {
padding: 5px 0;
}
.invoice .invoice-items {
width: 100%;
}
.invoice .invoice-items td {
border-top: #eee 1px solid;
}
.invoice .invoice-items .total td {
border-top: 2px solid #333;
border-bottom: 2px solid #333;
font-weight: 700;
}
/* -------------------------------------
RESPONSIVE AND MOBILE FRIENDLY STYLES
------------------------------------- */
@media only screen and (max-width: 640px) {
body {
padding: 0 !important;
}
h1, h2, h3, h4 {
font-weight: 800 !important;
margin: 20px 0 5px !important;
}
h1 {
font-size: 22px !important;
}
h2 {
font-size: 18px !important;
}
h3 {
font-size: 16px !important;
}
.container {
padding: 0 !important;
width: 100% !important;
}
.content {
padding: 0 !important;
}
.content-wrap {
padding: 10px !important;
}
.invoice {
width: 100% !important;
}
}
</style>
</head>
<body itemscope itemtype="http://schema.org/EmailMessage">
<table class="body-wrap">
<tr>
<td></td>
<td class="container" width="600">
<div class="content">
<table class="main" width="100%" cellpadding="0" cellspacing="0">
<tr>
{{ if gt (len .Alerts.Firing) 0 }}
<td class="alert alert-warning">
{{ else }}
<td class="alert alert-good">
{{ end }}
{{ .Alerts | len }} alert{{ if gt (len .Alerts) 1 }}s{{ end }} for {{ range .GroupLabels.SortedPairs }}
{{ .Name }}={{ .Value }}
{{ end }}
</td>
</tr>
<tr>
<td class="content-wrap">
<table width="100%" cellpadding="0" cellspacing="0">
<tr>
<td class="content-block">
<a href='{{ template "__alertmanagerURL" . }}' class="btn-primary">View in {{ template "__alertmanager" . }}</a>
</td>
</tr>
{{ if gt (len .Alerts.Firing) 0 }}
<tr>
<td class="content-block">
<strong>[{{ .Alerts.Firing | len }}] Firing</strong>
</td>
</tr>
{{ end }}
{{ range .Alerts.Firing }}
<tr>
<td class="content-block">
<strong>Labels</strong><br />
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
{{ if gt (len .Annotations) 0 }}<strong>Annotations</strong><br />{{ end }}
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
<a href="{{ .GeneratorURL }}">Source</a><br />
</td>
</tr>
{{ end }}
{{ if gt (len .Alerts.Resolved) 0 }}
{{ if gt (len .Alerts.Firing) 0 }}
<tr>
<td class="content-block">
<br />
<hr />
<br />
</td>
</tr>
{{ end }}
<tr>
<td class="content-block">
<strong>[{{ .Alerts.Resolved | len }}] Resolved</strong>
</td>
</tr>
{{ end }}
{{ range .Alerts.Resolved }}
<tr>
<td class="content-block">
<strong>Labels</strong><br />
{{ range .Labels.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
{{ if gt (len .Annotations) 0 }}<strong>Annotations</strong><br />{{ end }}
{{ range .Annotations.SortedPairs }}{{ .Name }} = {{ .Value }}<br />{{ end }}
<a href="{{ .GeneratorURL }}">Source</a><br />
</td>
</tr>
{{ end }}
</table>
</td>
</tr>
</table>
<div class="footer">
<table width="100%">
<tr>
<td class="aligncenter content-block"><a href='{{ .ExternalURL }}'>Sent by {{ template "__alertmanager" . }}</a></td>
</tr>
</table>
</div></div>
</td>
<td></td>
</tr>
</table>
</body>
</html>
