实现基于prometheus联邦收集node的指标数据

阅读目录

•   Prometheus-Server 联邦集群环境:
•   安装prometheus-server（主节点与联邦节点均部署）
•   部署node-exporter
•    配置联邦 server 监控 node_exporter
•   prometheus server 采集联邦 server

回到顶部

  Prometheus-Server 联邦集群环境:

  prometheus 主节点

  prometheus-1 联邦节点1

  prometheus-2 联邦节点2

  node1 联邦节点1的目标采集服务器

  node2 联邦节点2的目标采集服务器

 

回到顶部

  安装prometheus-server（主节点与联邦节点均部署）

  每个节点服务器安装的prometheus-server服务版本要保持一致

root@prometheus-2:~# tar xf prometheus-2.38.0.linux-amd64.tar.gz
root@prometheus-1:/usr/local# ln -s prometheus-2.38.0.linux-amd64 prometheus

 

  创建prometheus service启动脚本：默认监听9090端口

cat > /etc/systemd/system/prometheus.service << EOF
[Unit]
Description=Prometheus Server
Documentation=https://prometheus.io/docs/introduction/overview/
After=network.target
 
[Service]
Restart=on-failure
WorkingDirectory=/usr/local/prometheus/
ExecStart=/usr/local/prometheus/prometheus --config.file=/usr/local/prometheus/prometheus.yml --web.enable-lifecycle 
--storage.tsdb.retention=720h
 
[Install]
WantedBy=multi-user.target
EOF

 

  启动prometheus

systemctl daemon-reload && systemctl enable prometheus.service && systemctl restart prometheus.service

 

  prometheus-1 验证

 

  prometheus-2 验证

 

回到顶部

  部署node-exporter

   部署node-exporter

root@node1:/usr/local/src# tar xf node_exporter-1.3.1.linux-amd64.tar.gz -C /usr/local/
root@node1:/usr/local# ln -s node_exporter-1.3.1.linux-amd64 node_exporter

  

  创建node-exporter service启动文件

root@node1:~# vim /etc/systemd/system/node-exporter.service
[Unit]
Description=Prometheus Node Exporter
After=network.target
 
[Service]
ExecStart=/usr/local/node_exporter/node_exporter
 
[Install]
WantedBy=multi-user.target

 

  启动node-exporter

root@node1:~# systemctl daemon-reload && systemctl restart node-exporter && systemctl enable node-exporter.service

 

  验证node-exporter1

 

  验证node-exporter2

 

回到顶部

   配置联邦 server 监控 node_exporter

   分别在联邦节点 1 监控 node1，在联邦节点 2 监控 node2

 

  联邦prometheus-server1节点配置：

root@prometheus-1:/usr/local/prometheus# vim prometheus.yml
  - job_name: "prometheus-node1"
    static_configs:
      - targets: ["192.168.100.22:9100"]

 

  联邦prometheus-server2节点配置：

root@prometheus-2:/usr/local/prometheus# vim prometheus.yml 
  - job_name: "prometheus-node1"
    static_configs:
      - targets: ["192.168.100.23:9100"]

 

  重启prometheus-server

systemctl restart prometheus.service 

 

  验证

  prometheus-server1

 

  prometheus-server2

 

回到顶部

  prometheus server 采集联邦 server

  采集二进制prometheus-server联邦节点

  配置prometheus-server，添加采集prometheus联邦节点prometheus-1 和 prometheus-2

root@prometheus:/usr/local/prometheus# vim prometheus.yml  
  - job_name: "prometheus-federate-1"
    scrape_interval: 10s
    honor_labels: true
    metrics_path: '/federate'  #指定联邦URI
    params:
      'match[]':    #获取所有lable数据
       - '{job="prometheus"}'   #匹配prometheus job
       - '{__name__=~"job:.*"}'  #匹配所有job
       - '{__name__=~"node.*"}'  #获取所哟node
    static_configs:
      - targets: ["192.168.100.17:9090","192.168.100.18:9090"]  #指定联邦及诶单prometheus-1

 

  验证联邦节点

 

  访问联邦节点URI /federate

 

  prometheus-sever验证联邦节点promql查询监控的node节点数据

 

  grafana查看node-exporter的dashboard，数据源为prometheus-server主节点 ，模板ID:11074

 

  采集k8s prometheus-server联邦节点

  验证k8s prometheus

root@master1:~# kubectl get pods
NAME                                 READY   STATUS    RESTARTS         AGE
cadvisor-7l2tc                       1/1     Running   12 (4m26s ago)   24d
cadvisor-mp5nk                       1/1     Running   12 (4m25s ago)   24d
cadvisor-xlx9j                       1/1     Running   12 (4m24s ago)   24d
node-exporter-8w9fp                  1/1     Running   18 (4m25s ago)   27d
node-exporter-lr8lx                  1/1     Running   18 (4m26s ago)   27d
node-exporter-rx48k                  1/1     Running   18 (4m24s ago)   27d
prometheus-server-84c8755ccc-vtvfl   1/1     Running   8 (4m25s ago)    21d
 
root@master1:~# kubectl get svc
NAME         TYPE       CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
prometheus   NodePort   10.100.16.28   <none>        9090:30090/TCP   26d

 

root@master1:~# ifconfig ens33
ens33: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.100.3  netmask 255.255.255.0  broadcast 192.168.100.255
        inet6 fe80::20c:29ff:fe0f:4d56  prefixlen 64  scopeid 0x20<link>
        ether 00:0c:29:0f:4d:56  txqueuelen 1000  (Ethernet)
        RX packets 422652  bytes 231903817 (231.9 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 497685  bytes 364343660 (364.3 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

 

 

   配置prometheus-server主节点，添加k8s prometheus-server联邦节点job

root@prometheus:/usr/local/prometheus# vim prometheus.yml  
  - job_name: "kubernetes-federate"
    scrape_interval: 10s
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
       - '{job="prometheus"}'
       - '{__name__=~"job:.*"}'
       - '{__name__=~.*"}'
    static_configs:
      - targets: ["192.168.100.3:30090"]   #指定kubernetes的prometheus地址

 

  验证prometheus-server添加k8s联邦节点

 

 

 

  配置granfana，添加dashboard