prometheus添加集群

平台为MCNP.添加纽约地区集群.
参考:https://kubernetes.io/zh/docs/reference/access-authn-authz/authentication/
获取集群token

cat /root/.kube/config

node添加监控

1.在A集群master节点上备份prometheus的configmap文件，执行如下命令；

kubectl -n tcnp get cm prometheus-server1 -o yaml > /tmp/prometheus-server1-cm.yaml.bak

2.在线修改prometheus的configmap文件，执行如下命令；

kubectl -n tcnp edit cm prometheus-server1

添加C集群

    - job_name: kubernetes-apiservers-cls-cluster-nygp6by2
      honor_timestamps: false
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: endpoints
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        regex: default;kubernetes;https
        action: keep
      - target_label: __address__
        replacement: 192.168.182.11:6443
    - job_name: kubernetes-nodes-cls-cluster-dJ2lQ7jY4
      honor_timestamps: false
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: node
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - regex: __meta_kubernetes_node_label_(.+)
        action: labelmap
      - target_label: __address__
        replacement: 192.168.182.11:6443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics
    - job_name: kubernetes-nodes-cadvisor-cls-cluster-nygp6by2
      honor_timestamps: false
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: node
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - regex: __meta_kubernetes_node_label_(.+)
        action: labelmap
      - target_label: __address__
        replacement: 192.168.182.11:6443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    - job_name: kubernetes-service-endpoints-cls-cluster-nygp6by2
      honor_timestamps: false
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: endpoints
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_label_component]
        regex: ^(node-exporter|kube-state-metrics)$
        action: keep
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        regex: "true"
        action: keep
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        regex: (https?)
        target_label: __scheme__
        action: replace
      - source_labels: [__meta_kubernetes_pod_node_name, __meta_kubernetes_pod_container_port_number]
        regex: ([^;]+);([^;]+)
        target_label: instance
        replacement: ${1}:${2}
        action: replace
      - target_label: __address__
        replacement: 192.168.182.11:6443
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_name, __meta_kubernetes_pod_container_port_number]
        regex: ([^;]+);([^;]+);([^;]+)
        target_label: __metrics_path__
        replacement: /api/v1/namespaces/${1}/pods/http:${2}:${3}/proxy/metrics
      - regex: __meta_kubernetes_service_label_(.+)
        action: labelmap
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
        action: replace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name
        action: replace
    - job_name: tke-service-endpoints-cls-cluster-nygp6by2
      honor_timestamps: false
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: endpoints
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_tke_prometheus_io_scrape]
        regex: "true"
        action: keep
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        regex: (https?)
        target_label: __scheme__
        action: replace
      - source_labels: [__meta_kubernetes_pod_node_name, __meta_kubernetes_pod_container_port_number]
        regex: ([^;]+);([^;]+)
        target_label: instance
        replacement: ${1}:${2}
        action: replace
      - target_label: __address__
        replacement: 192.168.182.11:6443
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_name, __meta_kubernetes_pod_container_port_number]
        regex: ([^;]+);([^;]+);([^;]+)
        target_label: __metrics_path__
        replacement: /api/v1/namespaces/${1}/pods/http:${2}:${3}/proxy/metrics
      - regex: __meta_kubernetes_service_label_(.+)
        action: labelmap
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
        action: replace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name
        action: replace
    - job_name: kubernetes-pods-cls-cluster-nygp6by2
      honor_timestamps: false
      scrape_interval: 1m
      scrape_timeout: 10s
      metrics_path: /metrics
      scheme: https
      kubernetes_sd_configs:
      - api_server: https://192.168.182.11:6443
        role: pod
        bearer_token: sQ8nG6mC0eP5sL9d
        tls_config:
          insecure_skip_verify: true
      bearer_token: sQ8nG6mC0eP5sL9d
      tls_config:
        insecure_skip_verify: true
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        separator: ;
        regex: ^(redis-ha|redis-cluster)$
        replacement: $1
        action: keep
      - source_labels: [__meta_kubernetes_pod_container_name]
        separator: ;
        regex: ^(redis-exporter)$
        replacement: $1
        action: keep
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        separator: ;
        regex: "true"
        replacement: $1
        action: keep
      - source_labels: [__meta_kubernetes_pod_node_name, __meta_kubernetes_pod_container_port_number]
        regex: ([^;]+);([^;]+)
        target_label: instance
        replacement: ${1}:${2}
        action: replace
      - separator: ;
        regex: (.*)
        target_label: __address__
        replacement: 192.168.182.11:6443
        action: replace
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_name, __meta_kubernetes_pod_annotation_prometheus_io_port,
          __meta_kubernetes_pod_annotation_prometheus_io_path]
        separator: ;
        regex: ([^;]+);([^;]+);([^;]+);([^;]+)
        target_label: __metrics_path__
        replacement: /api/v1/namespaces/${1}/pods/http:${2}:${3}/proxy${4}
        action: replace
      - separator: ;
        regex: __meta_kubernetes_pod_label_(.+)
        replacement: $1
        action: labelmap
      - source_labels: [__meta_kubernetes_namespace]
        separator: ;
        regex: (.*)
        target_label: kubernetes_namespace
        replacement: $1
        action: replace
      - source_labels: [__meta_kubernetes_pod_name]
        separator: ;
        regex: (.*)
        target_label: kubernetes_pod_name
        replacement: $1
        action: replace

3.检查是否添加成功，执行如下命令；

kubectl -n kube-system describe cm prometheus-server1

4.在Grafana页面检查是否有C集群

master添加监控

由于master为不可调度,需要设置promutheus容器容忍度到不可调度的
1.在C集群master节点上备份prometheus-node-exporter的daemonset文件；
2.在线修改prometheus-node-exporter的daemonset，执行如下命令；

kubectl -n kube-system edit ds prometheus-node-exporter

添加这三行即可

terminationGracePeriodSeconds: 30
      tolerations:
      - effect: NoSchedule
        operator: Exists
      volumes:

3.执行kubectl -nkube-system get po | grep prometheus | wc -l检查是否增加