prometheus-webhook-grafana-alertmanager-consul

install docker#

yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum makecache fast
yum -y install docker-ce
systemctl start docker

touch /etc/docker/daemon.json
cat >> /etc/docker/daemon.json << EOF
{
    "registry-mirrors": ["http://hub-mirror.c.163.com","https://docker.mirrors.ustc.edu.cn","https://registry.docker-cn.com"]
}

EOF
systemctl restart docker
docker info

install docker images#

docker pull prom/prometheus
docker pull grafana/grafana
docker pull prom/alertmanager
docker pull timonwong/prometheus-webhook-dingtalk
docker pull consul

create files#

mkdir -p /home/prometheus/prometheus/data
chmod -R 777 /home/prometheus/prometheus/data
mkdir -p /home/prometheus/grafana
mkdir -p /home/prometheus/alertmanager/data
mkdir -p /home/prometheus/webhook-dingtalk

start grafana#

docker run -d -p 3000:3000 \
--user=root \
--name=grafana \
-v /home/prometheus/grafana:/var/lib/grafana \
grafana/grafana
#--user=root  #以root用户运行
#grafana模板地址https://grafana.com/grafana/dashboards

start prometheus#

docker run  -d -p 9090:9090 --name prometheus prom/prometheus
docker cp prometheus:/etc/prometheus/prometheus.yml /home/prometheus/prometheus/
docker rm -f prometheus

docker run  -d -p 9090:9090 \
--name prometheus \
-v /home/prometheus/prometheus:/etc/prometheus \
-v /home/prometheus/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
prom/prometheus \
--web.enable-lifecycle \
--config.file="/etc/prometheus/prometheus.yml" \
--storage.tsdb.path="/etc/prometheus/data"
#--web.enable-lifecycle							#热加载参数，需要配合配置文件--config.file使用，否则会报错
#curl -X POST http://localhost:9090/-/reload	#热加载prometheus配置文件
#--config.file									#配置文件路径
#--storage.tsdb.path="/etc/prometheus/data"		#数据存储路径

start alertmanager#

docker run --name alertmanager -d -p 9093:9093 prom/alertmanager
docker cp alertmanager:/etc/alertmanager/alertmanager.yml /home/prometheus/alertmanager/
docker rm -f alertmanager

docker run -d --name alertmanger -p 9093:9093 \
-v /home/prometheus/alertmanager:/etc/alertmanager \
-v /home/prometheus/alertmanager/data:/etc/alertmanager/data \
prom/alertmanager \
--storage.path="/etc/alertmanager/data" \
--config.file="/etc/alertmanager/alertmanager.yml"
#--storage.path   数据存储路径
#--config.file    配置文件路径

connect alertmanagers+rules+grafana+prometheus#

cat > /home/prometheus/prometheus/prometheus.yml << EOF
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
           - 192.168.1.164:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
- /etc/prometheus/rules/*.rules
# - "first_rules.yml"
# - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'grafana'     #任务名 全局唯一
    scrape_interval: 5s     # 抓取采样数据的时间间隔
    static_configs:         #静态目标配置
      - targets: ['192.168.1.164:3000']   #抓取地址，默认为/metrics
        labels:             #标签
          instance: grafana
		  
EOF

docker restart prometheus
curl -X POST http://localhost:9090/-/reload

start webhook#

docker run -d --name webhook timonwong/prometheus-webhook-dingtalk
docker cp webhook:/etc/prometheus-webhook-dingtalk/config.yml /home/prometheus/webhook-dingtalk/
docker rm -f webhook

docker run -d \
-p 8060:8060 \
--name webhook \
-v /home/prometheus/webhook-dingtalk/config.yml:/etc/prometheus-webhook-dingtalk/config.yml \
timonwong/prometheus-webhook-dingtalk 

sed -i "/webhook1:/{n;s/url.*/url: https:\/\/oapi.dingtalk.com\/robot\/send?access_token=04124dfa8d5003019231b46aa7b17d3c027fba52113bb/g}" /home/prometheus/webhook-dingtalk/config.yml

connect alertmanager+webhook#

cat > /home/prometheus/alertmanager/alertmanager.yml << EOF
global:
  resolve_timeout: 5m
  smtp_from: '70267@qq.com'
  smtp_smarthost: 'smtp.qq.com:465'
  smtp_auth_username: '70267@qq.com'
  smtp_auth_password: 'pqkldsftqmcsbbbg'
  smtp_require_tls: false
  smtp_hello: 'qq.com'
route:
  group_by: ['alertname']
  group_wait: 5s
  group_interval: 5s
  repeat_interval: 24h
  receiver: 'webhook'
receivers:
- name: 'email'
  email_configs:
  - to: '70267@qq.com'
- name: 'webhook'
  webhook_configs:
  - url: 'http://192.168.1.164:8060/dingtalk/webhook1/send'
    send_resolved: true 
inhibit_rules:
  - source_match:
      serverity: 'critical'
    target_match:
      serverity: 'warning'
    equal: ['alertname','dev','instance']
EOF

docker restart alertmanger

start consul#

docker run --name consul -d -p 8500:8500 consul
#注销服务
#curl -X PUT http://192.168.1.164:8500/v1/agent/service/deregister/node-exporter
#node-exporter就是"id": "node-exporter"

connect consul+prometheus#

cat >> /home/prometheus/prometheus/prometheus.yml << EOF
  - job_name: 'node_exporter'
    consul_sd_configs:
      - server: '192.168.1.164:8500'
        services: [] 
    relabel_configs:
      - source_labels: [__meta_consul_tags]
        regex: .*node_exporter.*
        action: keep
      - regex: __meta_consul_service_metadata_(.+)
        action: labelmap

  - job_name: 'redis_exporter'
    consul_sd_configs:
      - server: '192.168.1.164:8500'
        services: []
    relabel_configs:
      - source_labels: [__meta_consul_tags]
        regex: .*redis_exporter.*
        action: keep
      - regex: __meta_consul_service_metadata_(.+)
        action: labelmap

  - job_name: 'mysqld_exporter'
    consul_sd_configs:
      - server: '192.168.1.164:8500'
        services: []
    relabel_configs:
      - source_labels: [__meta_consul_tags]
        regex: .*mysqld_exporter.*
        action: keep
      - regex: __meta_consul_service_metadata_(.+)
        action: labelmap

EOF

curl -X POST http://localhost:9090/-/reload

----------------------以下为客户端执行----------

客户端注册服务 到 192.168.1.164:8500#

id=scjpf_qt			#scjpf_qt
addr=1.2.3.4		#客户端IP
company=分组名称	#名称
server=前台		#前台/后台

mkdir -p /home/prometheus/consul/conf
mkdir /home/prometheus/consul/node_exporter
mv node_exporter /home/prometheus/consul/node_exporter/
chmod +x /home/prometheus/consul/node_exporter/node_exporter

cat > /etc/systemd/system/node_exporter.service <<EOF
[Unit]
Description=node_exporter
After=network.target

[Service]
Type=simple
ExecStart=/home/prometheus/consul/node_exporter/node_exporter
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

systemctl start node_exporter.service
systemctl status node_exporter.service
systemctl enable node_exporter.service


cat > /home/prometheus/consul/conf/node_exporter.json <<EOF
{
  "ID":"node_exporter_${id}",
  "Name":"node_exporter_${company}_${server}",
  "Tags": [
    "node_exporter"
  ],
  "Address":"${addr}",
  "Port": 9100,
  "Meta": {
    "project":"外部项目",
    "team":"数智云",
    "company":"${company}",
    "app":"${company}_${server}_linux",
    "server":"${server}"
  },
  "EnableTagOverride":false,
  "Check": {
    "HTTP":"http://${addr}:9100/metrics",
    "Interval":"10s"
  },
  "Weights": {
    "Passing": 10,
    "Warning": 1
  }
}
EOF

cat >> /home/shell/consul.sh << EOF
#!/bin/bash
curl --request PUT --data @/home/prometheus/consul/conf/node_exporter.json http://服务端外网IP:65530/v1/agent/service/register?replace-existing-checks=1
EOF

删除监控#

####----------以下为服务端删除监控-------------
curl -X PUT http://192.168.1.164:8500/v1/agent/service/deregister/node_exporter_scjpf_qt

####----------以下为客户端删除监控-------------

systemctl stop node_exporter
rm -f /etc/systemd/system/node_exporter.service
rm -rf /home/consul/node_exporter
rm -rf /home/consul/conf/node_exporter.json