Prometheus Service Discovery

Service Discovery:

file discovery

prometheus.yaml

scrape_configs:
  - job_name: "prometheus"
    metrics_path: /metrics
    static_configs:
    file_sd_configs:
    - files:
      - targets/prometheus-*.yaml
      refresh_interval: 2m
  
  - job_name: "node_exporter"
    metrics_path: /metrics
    static_configs:
    file_sd_configs:
    - files:
      -targets/node-*.yaml
    refresh_interval: 2m

targets/prometheus-server.yaml

- targets:
  - ram9:9090
  labels:
    app: prometheus
    job: prometheus

targets/node-ram.yaml

- targets:
  - ram7:9100
  - ram8:9100
  - ram9:9100
  labels:
    app: node_exporter
    job: node_exporter

dns discovery

SRV

- job_name: "node_exporter"
    static_configs:
    file_sd_configs:
    dns_sd_configs:
    - names: ["_node-exporter._tcp.ram7.concerto.com", "_node-exporter._tcp.ram8.concerto.com", "_node-exporter._tcp.ram9.concerto.com"]
      type: SRV

  - job_name: "node_exporter"
    static_configs:
    file_sd_configs:
    dns_sd_configs:
    - names: ["ram7.concerto.com", "ram8.concerto.com", "ram9.concerto.com"]
      type: A
      port: 9100

DNS

ram7                          IN        A                 192.168.8.17
ram8                          IN        A                 192.168.8.18
ram9                          IN        A                 192.168.8.19
; _service._proto.name. TTL class RR priority weight port target
_node-exporter._tcp.ram7      IN        SRV 10 20 9100    ram7
_node-exporter._tcp.ram8      IN        SRV 10 20 9100    ram8
_node-exporter._tcp.ram9      IN        SRV 10 20 9100    ram9

consul
/etc/consul.d/consul.hcl

datacenter = "dc1"
data_dir = "/opt/consul"
client_addr = "0.0.0.0"
ui_config{
  enabled = true
}
server = true
bind_addr = "0.0.0.0" # Listen on all IPv4
bootstrap_expect=1

# https://learn.hashicorp.com/tutorials/consul/get-started-key-value-store?in=consul/getting-started
consul agent -dev -node machine
consul members
consul members -detailed
curl localhost:8500/v1/catalog/nodes
# localhost:8600 dns
dig @localhost -p 8600 machine.node.consul
consul leave

echo '{  "service": {    "name": "web1",    "tags": [      "rails"    ],    "port": 81  }}' > ./consul.d/web1.json
echo '{  "service": {    "name": "web2",    "tags": [      "rails"    ],    "port": 82  }}' > ./consul.d/web2.json
consul agent -dev -enable-script-checks -config-dir ./consul.d
dig @localhost -p 8600 web.service.consul
dig @localhost -p 8600 -t SRV web.service.consul
dig @localhost -p 8600 -t SRV _web._tcp.service.consul
dig @localhost -p 8600 rails.web.service.consul  # A
curl http://localhost:8500/v1/catalog/service/web
curl http://localhost:8500/v1/health/service/web?passing
# UI
consul agent -dev -ui -client 0.0.0.0 -config-dir consul.d
# KV
consul kv put redis/config/minconns 1
consul kv put -flags=42 redis/config/users/admin abcd1234
consul kv get redis/config/minconns
consul kv get -detailed redis/config/minconns
consul kv get -recurse
consul kv get -recurse redis/config
consul kv delete redis/config/maxconns
consul kv delete -recurse redis

consul agent -server -node node-virago -ui -syslog -data-dir /opt/consul -client 0.0.0.0 -config-dir consul.d -bootstrap -bootstrap-expect 1 -datacenter tactile -bind 192.168.8.41

# /etc/consul.d/node_exporters.json
{
  "services": [
    {
      "id": "node_exporter-ram7",
      "name": "ram7",
      "address": "192.168.8.17",
      "port": 9100,
      "tags": ["node_exporter"],
      "checks": [{
        "http": "http://192.168.8.17:9100/metrics",
        "interval": "5s"
      }]
    },
    {
      "id": "node_exporter-ram8",
      "name": "ram8",
      "address": "192.168.8.18",
      "port": 9100,
      "tags": ["node_exporter"],
      "checks": [{
        "http": "http://192.168.8.18:9100/metrics",
        "interval": "5s"
      }]
    },
    {
      "id": "node_exporter-ram9",
      "name": "ram9",
      "address": "192.168.8.19",
      "port": 9100,
      "tags": ["node_exporter"],
      "checks": [{
        "http": "http://192.168.8.19:9100/metrics",
        "interval": "5s"
      }]
    }
  ]
}

# /etc/consul.d/node_exporter.json
{
  "service": {
    "id": "node_exporter-ram7",
    "name": "ram7",
    "address": "192.168.8.17",
    "port": 9100,
    "tags": ["node_exporter", "linux"],
    "checks": [{
      "http": "http://192.168.8.17:9100/metrics",
      "interval": "5s"
    }]
  }
}

# /etc/consuld.d/prometheus.json
{
  "services": [
    {
      "id": "prometheus-server-id",
      "name": "prometheus-server-name",
      "address": "192.168.8.19",
      "port": 9090,
      "tags": ["prometheus"],
      "checks": [{
        "http": "http://192.168.8.19:9090/metrics",
        "interval": "5s"
      }]
    }
  ]
}

consul reload
consul services deregister -id node_exporter-ram8
consul services register nodex_exporters.json

prometheus.yml

 - job_name: "prometheus"
    static_configs:
    file_sd_configs:
    consul_sd_configs:
    - server: "ram6:8500"
      tags:
      - "prometheus"
      refresh_interval: 20s

  - job_name: "node_exporter"
    static_configs:
    file_sd_configs:
    consul_sd_configs:
    - server: "ram6:8500"
      tags:
      - "nodes"
      refresh_interval: 20s

Relabeling:

go_info

  - job_name: "node_exporter"
    static_configs:
    file_sd_configs:
    consul_sd_configs:
    - server: "ram6:8500"
      tags:
      - "nodes"
      refresh_interval: 20s
    relabel_configs:
    - source_labels:
      - __scheme__
      - __address__
      - __metrics_path__
      regex: "(http|https)(.*)"
      separator: ""
      target_label: "endpoint"
      replacement: "${1}://${2}"
      action: replace

    metric_relabel_configs:
    - source_labels:
      - __name__
      regex: "go_info.*"
      action: drop