prometheus之tar包部署

prometheus

1. 部署

下载软件github

 prometheus-2.14.0-rc.0.linux-amd64.tar.gz

解压

mkdir /opt/src
tar -xvf prometheus-2.14.0-rc.0.linux-amd64.tar.gz -C /opt/
cd /opt/
ln -s prometheus-2.14.0-rc.0.linux-amd64/ prometheus

添加用户

groupadd prometheus
useradd -g prometheus -s /sbin/nologin prometheus

创建数据目录+权限

chown -R prometheus:prometheus /opt/prometheus
mkdir -p /data/prometheus
chown -R prometheus:prometheus /data/prometheus

配置启动文件

v /usr/lib/systemd/system/prometheus.service

[Unit]

Description=Prometheus

Documentation=https://prometheus.io/

After=network.target

[Service]

# Type设置为notify时，服务会不断重启

Type=simple

User=prometheus
ExecStart=/opt/node_exporter/node_exporter --collector.systemd --collector.systemd.unit-whitelist=(docker|sshd).service --collector.textfile.directory /var/lib/node_exporter/textfile_collector/   

Restart=on-failure

[Install]

WantedBy=multi-user.target



chown prometheus:prometheus /usr/lib/systemd/system/prometheus.service

 

安装node_export

tar -xvf node_exporter-0.18.1.linux-amd64.tar.gz -C /opt/
cd /opt/
ln -s node_exporter-0.18.1.linux-amd64/ node_exporter

groupadd prometheus

useradd -g prometheus -s /sbin/nologin prometheus

chown -R prometheus:prometheus /opt/node_exporter

 


vi /usr/lib/systemd/system/node_exporter.service

[Unit]

Description=node_exporter

Documentation=https://prometheus.io/

After=network.target

[Service]

Type=simple

User=prometheus

ExecStart=/opt/node_exporter/node_exporter --collector.systemd.unit-whitelist="(docker|sshd).service" --collector.textfile.directory /var/lib/node_exporter/textfile_collector/ 

Restart=on-failure

[Install]

WantedBy=multi-user.target

 

systemctl enable node_exporter

systemctl start node_exporter


mkdir -p /var/lib/node_exporter/textfile_collector

cat /var/lib/node_exporter/textfile_collector/metadata.prom 
metadata{role="docker_server",datacenter="NJ"} 1

 

prometheus配置文件添加

   - job_name: 'node'
     static_configs:
     - targets: ['10.4.7.11:9100'] 

热重启prometheus

kill -HUP <pid>

node节点运行cadvisor

docker run \
  --volume=/:/rootfs:ro \
  --volume=/var/run:/var/run:rw \
  --volume=/sys:/sys:ro \
  --volume=/var/lib/docker/:/var/lib/docker:ro \
  --volume=/dev/disk/:/dev/disk:ro \
  --publish=8080:8080 \
  --detach=true \
  --name=cadvisor \
  google/cadvisor:latest

再次添加prometheus配置

   - job_name: 'docker'
     static_configs:
      - targets: ['10.4.7.11:8080']

添加rules规则文件，在prometheus.yml同级目录创建

[root@localhost prometheus]# mkdir rules
[root@localhost prometheus]# cd rules/
[root@localhost rules]# touch node_rules.yml

修改prometheus.yml文件

rule_files:
  - "rules/node_rules.yml"

node_rules.yml文件添加规则

groups:
- name: node_rules
  rules:
  - record: instance:node_cpu:avg_rate5m
    expr: 100 - avg (irate(node_cpu_seconds_total{job="node",mode="idle"}[5m])) by (instance) * 100
  - record: instance:node_cpus:count
    expr: count by (instance)(node_cpu_seconds_total{mode="idle"})
  - record: instance:node_cpu_saturation_load1
    expr: node_load1 > on (instance) 2 * count by (instance)(node_cpu_seconds_total{mode="idle"})
  - record: instance:node_memory_usage:percentage
    expr: (node_memory_MemTotal_bytes - (node_memory_MemFree + node_memory_Cached_bytes + node_memory_Buffers_bytes)) / node_memory_MemTotal_bytes * 100
  - record: instance:node_memory_swap_io_bytes:sum_rate
    expr: 1024 * sum by (instance) (
                 (rate(node_vmstat_pgpgin[1m])
                 + rate(node_vmstat_pgpgout[1m]))
          )
  - record: instance:root:node_filesystem_usage:percentage
    expr: (node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) / node_filesystem_size_bytes{mountpoint="/"} * 100

添加完后可通过promtools监测文件

./promtool check rules rules/node_rules.yml 

监测通过后需要重载prometheus配置文件

kill -HUP 2018

在prom页面中即可搜索到键值

instance:node_cpu:avg_rate5m

下载安装granafa

wget https://dl.grafana.com/oss/release/grafana-6.7.1-1.x86_64.rpm

yum -y install grafana-6.7.1-1.x86_64.rpm 

systemctl start grafana-server.service