Prometheus - 基础监控服务端的搭建

1. 环境和部署内容

　　服务器系统：CentOS Linux release 7.6.1810]()
　　基于Docker搭建Prometheus、Grafana、node-exporter，使用InfluxDB存储监控数据。

2. 安装Docker

感觉华为云mirror速度快一些

wget -O /etc/yum.repos.d/docker-ce.repo https://mirrors.huaweicloud.com/docker-ce/linux/centos/docker-ce.repo
sudo sed -i 's+download.docker.com+mirrors.huaweicloud.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo

sudo yum makecache fast
sudo yum install docker-ce

3. 安装InfluxDB

直接从官网下载安装

# 官网：https://portal.influxdata.com/downloads/
wget https://dl.influxdata.com/influxdb/releases/influxdb-1.8.3.x86_64.rpm
sudo yum localinstall influxdb-1.8.3.x86_64.rpm
systemctl start influxdb
systemctl enable influxdb

配置用户及认证

# 执行influxdb登录数据库创建管理员用户
influxdb
CREATE USER "admin" WITH PASSWORD 'admin' WITH ALL PRIVILEGES

# 需要修改配置文件，开启认证
vim /etc/influxdb/influxdb.conf
auth-enabled = true

# 修改完配置重启数据库
systemctl restart influxdb

尝试认证登录，创建数据库、用户并授权

influx --username admin -password admin
CREATE DATABASE prometheus
CREATE USER "prometheus" WITH PASSWORD 'prometheus'
grant all on prometheus to prometheus

检查用户

influx --username prometheus -password prometheus
show databases
# 能看到prometheus就可以了

4. 安装Prometheus、Grafana、node-exporter

# pull！就完事了！
docker pull prom/node-exporter
docker pull prom/prometheus
docker pull grafana/grafana

5. 配置Prometheus

# 创建目录
mkdir -p /opt/prometheus/fileconfig/
mkdir /opt/prometheus/data/
chmod 777 /opt/prometheus/data/

vim /opt/prometheus/prometheus.yml

# my global config
global:
  scrape_interval:     60s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 60s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
  - static_configs:
    - targets:
      # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: 'prometheus'

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.
    # 这台是本机的node-exporter地址
    static_configs:
    - targets: ['192.168.11.11:9100']

  # 用于动态增加监控地址
  - job_name: 'remote-server'
    scrape_interval: 60s
    static_configs:
    honor_labels: true
    file_sd_configs:
      - files:
        - '/etc/prometheus/fileconfig/*-nodes.json'

# 数据写入到InfluxDB
remote_write:
  - url: "http://192.168.11.11:8086/api/v1/prom/write?db=prometheus&u=prometheus&p=prometheus"
remote_read:
  - url: "http://192.168.11.11:8086/api/v1/prom/read?db=prometheus&u=prometheus&p=prometheus"

5. 启动服务

# 启动Prometheus
# --web.enable-admin-api会开启HTTP API，如暴露在互联网，注意安全问题
# --storage.tsdb.path=/etc/prometheus/data 指定prometheus的数据存储路径
# --storage.tsdb.retention.time 设置本地存储的过期时间（虽然数据会远程写入到influxdb，但是prometheus还是会在本地存储数据）

docker run -d \
 --name prometheus \
 -p 9090:9090 \
 -v /opt/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml \
 -v /opt/prometheus/fileconfig:/etc/prometheus/fileconfig \
 -v /opt/prometheus/data:/etc/prometheus/data \
 --restart=always \
 prom/prometheus \
 --config.file=/etc/prometheus/prometheus.yml \
 --web.enable-admin-api \
 --storage.tsdb.path=/etc/prometheus/data \
 --storage.tsdb.retention.time 12h

# 启动node-exporter（多个磁盘全都要-v进去）
docker run -d \
 -p 9100:9100 \
 -v "/proc:/host/proc:ro" \
 -v "/sys:/host/sys:ro" \
 -v "/:/rootfs:ro" \
 --net="host" \
 prom/node-exporter

# 启动grafana
docker run -d \
 -p 3000:3000 \
 --name=grafana \
 -v /opt/grafana-storage:/var/lib/grafana \
 grafana/grafana

6. 验证功能

　　检查node-exporter的数据
　　　　http://192.168.11.11:9100/metrics
　　检查prometheus是否获取target，需要等一小会儿抓到metric才会UP
　　　　http://192.168.11.11:9090/targets
　　访问Grafana（admin/admin）
　　　　http://192.168.11.11:3000

　　下载、导入Grafana的Dashboard看看效果：https://grafana.com/grafana/dashboards/8919