Install:

tar xvzf prometheus-2.36.1.linux-amd64.tar.gz --directory /usr/local
cd /usr/local
ln -fnsv prometheus-2.36.1.linux-amd64 prometheus
groupadd --gid 9090 --system prometheus
useradd --uid 9090 --gid prometheus --shell /sbin/nologin --no-create-home --comment 'Prometheus User' --system --user-group prometheus
mkdir -pv --mode=700 /var/lib/prometheus
chown -HLR --dereference prometheus:prometheus /var/lib/prometheus
chown -HLR --dereference prometheus:prometheus /usr/local/prometheus

Consul Unit:

[Unit]
Description="HashiCorp Consul - A service mesh solution"
Documentation=https://www.consul.io/
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/etc/consul.d/consul.hcl

[Service]
EnvironmentFile=/etc/consul.d/consul.env
User=consul
Group=consul
ExecStart=/usr/bin/consul agent -config-dir=/etc/consul.d/
ExecReload=/bin/kill --signal HUP $MAINPID
KillMode=process
KillSignal=SIGTERM
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

Prometheus:

cat > /etc/systemd/system/prometheus.service <<'EOF'
[Unit]
Description=Prometheus
Documentation=https://prometheus.io
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=/usr/local/prometheus/prometheus \
  --config.file /usr/local/prometheus/prometheus.yml \
  --storage.tsdb.path /var/lib/prometheus \
  --web.console.templates /usr/local/prometheus/consoles \
  --web.console.libraries /usr/local/prometheus/console_libraries
ExecReload=/bin/kill -SIGHUP $MAINPID
TimeoutStartSec=20s
TimeoutStopSec=20s
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

node_exporter:

cat > /etc/systemd/system/node_exporter.service <<'EOF'
[Unit]
Description=Prometheus node_exporter
Documentation=https://prometheus.io
After=network.target
Wants=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=/usr/local/bin/node_exporter \
  --collector.ntp \
  --collector.mountstats \
  --collector.systemd \
  --collector.tcpstat
ExecReload=/bin/kill -SIGHUP $MAINPID
Restart=on-failure
KillMode=process
KillSignal=SIGTERM
TimeoutStartSec=20s
TimeoutStopSec=20s

[Install]
WantedBy=multi-user.target
EOF

mysqld_exporter:

[Unit]
Description=Prometheus mysqld_exporter
Documentation=https://prometheus.io
After=network.target
Wants=network.target

[Service]
Type=simple
User=prometheus
Group=prometheus
ExecStart=/usr/local/bin/mysqld_exporter \
  --config.my-cnf=my.cnf
ExecReload=/bin/kill -SIGHUP $MAINPID
TimeoutStartSec=20s
TimeoutStopSec=20s
Restart=on-failure

[Install]
WantedBy=multi-user.target

my.cnf:

my.cnf
[client]
host=127.0.0.1
user=mysqld_exporter
password=mysqld _exporter

授权mysqld_exporter
GRANT process,replication client,select ON *.* TO 'mysqld_exporter'@'localhost';

API:

https://prometheus.io/docs/prometheus/latest/querying/api/

// 删除某个组下某个实例的所有数据

curl -X DELETE http://192.168.0.10:9091/metrics/job/pushgateway-microservice/instance/pushgateway-microservice

// 删除某个组下的所有数据

curl -X DELETE http://192.168.0.10:9091/metrics/job/pushgateway-microservice

删除全部数据
curl -X POST -g 'http://192.168.0.10:9090//api/v1/admin/tsdb/delete_series?match[]={__name__=~".+"}'

启动
./prometheus --storage.tsdb.retention=180d --web.enable-admin-api

//清理db
curl -XPOST http://192.168.0.10:9090/api/v1/admin/tsdb/clean_tombstones

Grafana:

默认账户密码: admin@admin

模板地址：Grafana模板

Import >> 在Grafana.com Dashboard填入ID-8919（Linux系统监控模板）

grafana 添加 prometheus 数据源

Configure - DataSources - Add data source 选择 Prometheus
HTTP URL 填写 http://IP:9090 （Prometheus对应的IP和端口）
点击 Save & Test 提交
Dashboards 选择 Prometheus 2.0 stats - Import
Search - General - Prometheus 2.0 Stats 打开图表查看

导入grafana模板，数据展示（grafana模板官网 https://grafana.com/dashboards）
打开 http://localhost:3000/
Create - Import - Import via grafana.com
输入9276号模板 - Load - 选择 Prometheus数据源 - 点击Import提交

Search - General - 1主机基础监控（CPU，内存，磁盘，网络）
分组 选择：Node IP地址选择：localhost:9100
即可以显示数据