搭建prometheus监控系统

安装prometheus

sudo mkdir /docker/compose/prometheus -p
cd /docker/compose/prometheus
touch docker-compose.yml
#如果复制换行错误
:set paste

version: '3.7'
services:
   prometheus:
      image: prom/prometheus:latest
      container_name: prometheus
      restart: on-failure:1
      ports:
         - "9090:9090"
      volumes:
         - /docker/config/prometheus/:/etc/prometheus/
      command:
            - '--config.file=/etc/prometheus/prometheus.yml'
            - '--web.enable-admin-api'
            - '--web.enable-lifecycle'

创建配置文件

cd /docker/config/prometheus
touch prometheus.yml

# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 127.0.0.1:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
    - "rules/*.yml"
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]
      
  - job_name: "service_status_http"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorServiceTargets
        
  - job_name: "service_status_https"
    scheme: https
    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorServiceTargets?scheme=https     
     
     
  - job_name: 'http_status'  # 配置job名
    metrics_path: /probe     # 定义metric获取的路径
    params:
      module: [http_2xx]     # 这里就是我们在black_exporter中定义的模块名
    #file_sd_configs:         # 因需要监控的地址很多，我们这里将所有地址独立出来，后面会介绍该文件
    #  - files: 
    #    - '/etc/prometheus/job_web.yaml'
    #    refresh_interval: 30s # 30秒刷新一次，当有新的监控地址时，会自动加载进来不需要重启
    http_sd_configs:
      - url: http://127.0.0.1:5220/api/Monitor/GetMonitorSiteTargets
    relabel_configs:
      - source_labels: [__address__]  # 当前target的访问地址，比如监控百度则为 https://baidu.com
        target_label: __param_target  # __param是默认参数前缀，target为参数，这里可以理解为把__address__ 的值赋给__param_target，若监控百度，则target=https://baidu.com
      - source_labels: [__param_target]
        target_label: instance        # 可以理解为把__param_target的值赋给instance标签
      - target_label: __address__
        replacement: 127.0.0.1:9115 # web监控原本的target为站点的地址，但Prometheus不是直接去请求该地址，而是去请求black_exporter，故需要把目标地址替换为black_exporter的地址

docker-compose

安装docker-compose参考

# 2. 安装docker-compose
pip install docker-compose

docker-compose up -d

安装grafana

#先安装原版,拿配置文件
docker run -d --name grafana \
-p 3000:3000 \
grafana/grafana:latest


docker cp grafana:/etc/grafana/ /docker/config/
docker cp grafana:/var/lib/grafana/ /docker/volumes/

docker run -d --name grafana \
-p 3000:3000 \
-v /docker/config/grafana/:/etc/grafana/ \
-v /docker/volumes/grafana/:/var/lib/grafana/ \
grafana/grafana:latest


#如果权限不足
# chmod -R 777 . 


#如果登录报错 /docker/config/grafana/grafana.ini
[log]
# Either "console", "file", "syslog". Default is console and  file
# Use space to separate multiple modes, e.g. "console file"
mode = console file

# Either "debug", "info", "warn", "error", "critical", default is "info"
level = debug

添加datasource

安装alertManager

docker run --name alertmanager -d -p 9093:9093 --restart=always \
prom/alertmanager


docker cp alertmanager:/etc/alertmanager /docker/config/alertmanager
docker rm -f alertmanager

docker run --name alertmanager -d -p 9093:9093 --restart=always \
-v /docker/config/alertmanager/:/etc/alertmanager/ \
prom/alertmanager

http://10.0.17.178:9090/   #prometheus
http://10.0.17.178:3000/   #grafana
http://10.0.17.178:9093/   #altermessage

配置站点监控

参考

安装配置black_expoter

mkdir -p /docker/compose/prometheus/black_expoter
touch docker-compose.yml

#重启普罗米修斯
curl -XPOST http://127.0.0.1:9090/-/reload

#docker-compose.yml
version: '3.7'
services:
  blackbox_exporter:
    container_name: blackbox_exporter
    image: prom/blackbox-exporter:master
    volumes:
      - /docker/config/blackbox_exporter/config.yml:/etc/blackbox_exporter/config.yml
    ports:
      - 9115:9115

#config.yml
modules:
  http_2xx:      # 给模块取名，后面在Prometheus的配置文件中会用到该名称
    prober: http # 探针类型，探针有多种类型如http、tcp、icmp、dns，不同的探针具有不同的功能
    timeout: 5s  # 探针检测超时时间
    http:
      valid_status_codes: [] # 有效的状态码，默认为200，也可以自己定义，比如你的站点304也可能是正常的
      method: GET            # http使用get请求
      fail_if_body_not_matches_regexp: [] # 对返回结果进行正则匹配，如果未匹配成功则认为失败
      tls_config:
        insecure_skip_verify: true        # 不安全的https跳过确认，如某些证书不合法或者过期，如果你在浏览器访问，那浏览器会让你确认是否继续，这里也是类似的意思。