kubernetes 二进制部署的prometheus实现服务发现

  k8s外部实现pod发现

  使用外部网络单独部署的prometheus-server进行服务发现k8s中的资源

  在 namespace monitoring 创建服务发现账号 prometheus 并授权。

 

  k8s创建sa用户并授权集群权限

root@master1:~/yaml# cat outside-prom-rbac.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitoring

---
apiVersion: v1
kind: Secret
type: kubernetes.io/service-account-token
metadata:
  name: prometheus-token
  namespace: monitoring
  annotations:
    kubernetes.io/service-account.name: "prometheus"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - services
  - endpoints
  - pods
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - "extensions"
  resources:
    - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: monitoring

 

  查看创建的sa的secret

root@master1:~/yaml\# kubectl get secrets -n monitoring 
NAME               TYPE                                  DATA   AGE
prometheus-token   kubernetes.io/service-account-token   3      24m

 

  将 token 保存至 prometheus server 节点的 k8s.token 文件，后期用于权限验证。

  获取token

root@master1:~/yaml\# kubectl describe secrets -n monitoring prometheus-token

  prometheus-server添加token

root@prometheus:~\# vim /usr/local/prometheus/k8s.token

  prometheus-server添加服务发现配置

root@prometheus:/usr/local/prometheus# cat prometheus.yml
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          # - alertmanager:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  # - "first_rules.yml"
  # - "second_rules.yml"

# A scrape configuration containing exactly one endpoint to scrape:
# Here it's Prometheus itself.
scrape_configs:
  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    # scheme defaults to 'http'.

    static_configs:
      - targets: ["localhost:9090"]

#配置k8s API-Server 服务发现
  - job_name: 'kubernetes-apiserver'
    kubernetes_sd_configs:
    - role: endpoints
      api_server: https://192.168.100.3:6443
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /usr/local/prometheus/k8s.token
    scheme: https
    tls_config:
      insecure_skip_verify: true
    bearer_token_file: /usr/local/prometheus/k8s.token
    relabel_configs:
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: default;kubernetes;https
    #自定义替换发现的服务器端口、协议等.
    - source_labels: [__address__]
      regex: '(.*):6443'
      replacement: '${1}:9100'
      target_label: __address__
      action: replace
    - source_labels: [__scheme__]
      regex: https
      replacement: http
      target_label: __scheme__
      action: replace

#配置k8s node服务发现
  - job_name: 'kubernetes-nodes-monitor'
    scheme: http
    tls_config:
      insecure_skip_verify: true
    bearer_token_file: /usr/local/prometheus/k8s.token
    kubernetes_sd_configs:
    - role: node
      api_server: https://192.168.100.3:6443
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /usr/local/prometheus/k8s.token
    #node标签重写
    relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
        regex: '(.*)'
        replacement: '${1}'
        action: replace
        target_label: LOC
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
        regex: '(.*)'
        replacement: 'NODE'
        action: replace
        target_label: Type
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region]
        regex: '(.*)'
        replacement: 'K8S-test'
        action: replace
        target_label: Env
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)

#配置namespace下的pod服务发现
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
    - role: pod
      api_server: https://192.168.100.3:6443
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /usr/local/prometheus/k8s.token
      namespaces:
        names:
        - kube-system
        - kubernetes-dashboard
    relabel_configs:
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name

#指定Pod发现条件
  - job_name: 'kubernetes-conditions-pod'
    kubernetes_sd_configs:
    - role: pod
      api_server: https://192.168.100.3:6443
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /usr/local/prometheus/k8s.token
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__ 
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: ([^:]+)(?::\d+)?;(\d+)
      replacement: $1:$2
      target_label: __address__
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name
    - source_labels: [__meta_kubernetes_pod_label_pod_template_hash]
      regex: '(.*)'
      replacement: 'K8S-test'
      action: replace
      target_label: Env

 

 

  验证访问prometheus服务发现target：

  

  job_name: 'kubernetes-conditions-pod'中的__meta_kubernetes_pod_annotation_prometheus_io_scrape标签，由于pod中未包含这个注解，所以并没有发现，删掉这个注解的标签重写匹配，就会将pod动态发现

    #- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
    #  action: keep
    #  regex: true

 

  删掉prometheus_io_scrape标签重写的prometheus配置后，这时候访问prometheus就会发现job kubernetes-conditions-pod的pod