Prometheus之Alertmanager配置详解

配置介绍

alertmanager.yml


global:
  [ smtp_from: <tmpl_string> ] #发件人邮箱地址
  [ smtp_smarthost: <string> ] #邮箱SMTP地址
  [ smtp_auth_username: <string> ] #发件人登录用户名
  [ smtp_auth_password: <secret> ] #发件人登录密码或者授权码
  [ smtp_require_tls: <bool> | default = true ] #是否需要TLS协议，默认true
  [ wechat_api_url: <string> | default = "https://qyapi.weixin.qq.com/cgi-bin/" ] #企业微信API地址
  [ wechat_api_secret: <secret> ] #企业微信API secret
  [ wechat_api_corp_id: <string> ] #企业微信corp id
  [ resolve_timeout: <duration> | default = 5m ] #在指定时间内没有新的事件就发送恢复通知
 
route:  #route用来设置告警分发策略
  receiver: 'default-receiver' #设置接收人
  [ group_by: '[' <labelname>, ... ']' ] #采用那个标签来作为分组。
  [ group_wait: <duration> | default = 30s ] #组告警等待时间。在等待时间结束后，如果有同组告警一起发出。
  [ group_interval: <duration> | default = 5m ] #两组告警间隔时间。
  [ repeat_interval: <duration> | default = 4h ] #重复告警间隔时间，减少相同邮件的发送频率。
  
receivers: #通知接收者列表


inhibit_rules: #抑制规则

target_match:        #target_ 新告警信息匹配规则
  [ <labelname>: <labelvalue>, ... ]
target_match_re:
  [ <labelname>: <regex>, ... ]
target_matchers:
  [ - <matcher> ... ]

source_match:         # source_ 已存在的告警信息
  [ <labelname>: <labelvalue>, ... ]
source_match_re:
  [ <labelname>: <regex>, ... ]
source_matchers:
  [ - <matcher> ... ]


[ equal: '[' <labelname>, ... ']' ]   #新告警与已有告警二者之间需要满足的条件

#在有新的告警通知匹配到target_match和target_match_re规则时，若存在已经发出的告警通知满足source_matcn或者source_match_re的匹配条件，并且与发送的告警与新的告警中由equal定义的标签完全相同，则启动抑制机制，新的告警不会发送；

#severity（告警验证等级）：CRITICAL（危急）, WARNING（警告）, INFO（信息）

email_config

# Whether to notify about resolved alerts.
[ send_resolved: <boolean> | default = false ]

# The email address to send notifications to.
to: <tmpl_string>

# The sender's address.
[ from: <tmpl_string> | default = global.smtp_from ]

# The SMTP host through which emails are sent.
[ smarthost: <string> | default = global.smtp_smarthost ]

# The hostname to identify to the SMTP server.
[ hello: <string> | default = global.smtp_hello ]

# SMTP authentication information.
[ auth_username: <string> | default = global.smtp_auth_username ]
[ auth_password: <secret> | default = global.smtp_auth_password ]
[ auth_secret: <secret> | default = global.smtp_auth_secret ]
[ auth_identity: <string> | default = global.smtp_auth_identity ]

# The SMTP TLS requirement.
# Note that Go does not support unencrypted connections to remote SMTP endpoints.
[ require_tls: <bool> | default = global.smtp_require_tls ]

# TLS configuration.
tls_config:
  [ <tls_config> ]

# The HTML body of the email notification.
[ html: <tmpl_string> | default = '{{ template "email.default.html" . }}' ]
# The text body of the email notification.
[ text: <tmpl_string> ]

# Further headers email header key/value pairs. Overrides any headers
# previously set by the notification implementation.
[ headers: { <string>: <tmpl_string>, ... } ]

消息分类发送

alertmanager.yml

global:
  smtp_from: '1304995320@qq.com' 
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: '1304995320@qq.com'
  smtp_auth_password: 'xxxxxxxxx'
  smtp_hello: '@qq.com'
  smtp_require_tls: false 
route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5s
  repeat_interval: 1m
  receiver: 'web.hook'  #默认的告警通知
  routes:
  - receiver: 'dingding' #critical级别的消息发给钉钉
    group_wait:10s
    match_re:
      severity:critical
  - receiver: 'wechat' #info级别的消息发给企业微信
    group_wait:10s
    match_re:
      severity:info
receivers:
- name: 'web.hook'
  email_configs:
    - to: '1304995320@qq.com'
      send_resolved: true
- name: 'dingding'
  webhook_configs:
    - url: 'http://192.168.174.105:8060/dingtalk/webhook1/send'
      send_resolved: true
- name: 'wechat'
  wechat_configs:
    - corp_id: 'xxxxxxxxxxxxxx'
      to_party:2
      agent_id: 1000006
      api_secret: xxxxxxxxxxxxxxxx
      send_resolved: true
inhibit_rules:
  - source_match:
      alertname: InstanceDown
      severity: 'critical'
    target_match:
      alertname: InstanceDown
      severity: 'critical'
    equal: ['alertname', 'dev', 'instance']

自定义消息模板

定义微信模板

/usr/local/alertmanager/template/message_template.templ

{{ define "wechat.default.message" }}
{{ range $i,$alert :=.Alerts }}
===alertmanager 监控报警===
告警状态: {{ .Status }}
告警级别: {{ $alert.Labels.severity }}
告警类型: {{ $alert.Labels.alertname }}
告警应用: {{ $alert.Annotations.summary }}
告警主机: {{ $alert.Lables.instance }}
告警主题: {{ $alert.Annotations.summary }}
触发阈值: {{ $alert.Annotations.value }}
告警详情: {{ $alert.Annotations.description }}
触发时间: {{ $alert.StartsAt.Format "2006-01-02 15:04:05" }}

==========end===========
{{ end }}
{{ end }}

定义email模板

/usr/local/alertmanager/template/email_template.tmpl

{{ define "email.default.html" }}
{{ range $i, $alert := .Alerts }}

===alertmanager 监控报警=== <br>

告警状态: {{ .Status }} <br>
告警级别: {{ $alert.Labels.severity }} <br>
告警类型: {{ $alert.Labels.alertname }} <br>
告警应用: {{ $alert.Annotations.summary }} <br>
告警主机: {{ $alert.Labels.instance }} <br>
告警主题: {{ $alert.Annotations.summary }} <br>
触发阈值: {{ $alert.Annotations.value }} <br>
告警详情: {{ $alert.Annotations.description }} <br>
故障时间: {{ $alert.StartsAt.Local.Format "2006-01-02 15:04:05" }} <br>


===========end=========== <br>

{{ end }} <br>
{{ end }} <br>

alertermanager引用模板

alertmanager.yml

global:
  smtp_from: '1304995320@qq.com' 
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: '1304995320@qq.com'
  smtp_auth_password: 'xxxxxxxxx'
  smtp_hello: '@qq.com'
  smtp_require_tls: false 
  
templates:   #引用模板
  - 'template/*.tmpl'


route:
  group_by: ['alertname']
  group_wait: 30s
  group_interval: 5s
  repeat_interval: 1m
  receiver: 'web.hook'  #默认的告警通知
  routes:
  - receiver: 'dingding' #critical级别的消息发给钉钉
    group_wait:10s
    match_re:
      severity:critical
  - receiver: 'wechat' #info级别的消息发给企业微信
    group_wait:10s
    match_re:
      severity:info
receivers:
- name: 'web.hook'
  email_configs:
    - to: '1304995320@qq.com'
      html: '{{ template "email.default.html" . }}'
      headers: { Subject: "[WARN] 报警邮件" }
      send_resolved: true
- name: 'dingding'
  webhook_configs:
    - url: 'http://192.168.174.105:8060/dingtalk/webhook1/send'
      send_resolved: true
- name: 'wechat'
  wechat_configs:
    - corp_id: 'xxxxxxxxxxxxxx'
      to_party:2
      agent_id: 1000006
      api_secret: xxxxxxxxxxxxxxxx
      send_resolved: true
inhibit_rules:
  - source_match:
      alertname: InstanceDown
      severity: 'critical'
    target_match:
      alertname: InstanceDown
      severity: 'critical'
    equal: ['alertname', 'dev', 'instance']