自动化监控kubernetes_sd_configs
- 架构图
- 监控K8s集群Pod（kubelet集成了cadvisor，暴露接口）
promethues -> apiserver(192.168.2.60:6443) -> kubelet(cadvisor)
创建rbac
[root@master k8s-ftp]# cat rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups:
- ""
resources:
- nodes
- services
- endpoints
- pods
- nodes/proxy
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
- nodes/metrics
verbs:
- get
- nonResourceURLs:
- /metrics
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: kube-system
[root@master k8s-ftp]# kubectl apply -f rbac.yaml
serviceaccount/prometheus created
获取token
[root@master k8s-ftp]# kubectl get sa prometheus -n kube-system -o yaml|tail -2
secrets:
- name: prometheus-token-hx5h8
[root@master k8s-ftp]# kubectl describe secret prometheus-token-hx5h8 -n kube-system
Name: prometheus-token-hx5h8
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: prometheus
kubernetes.io/service-account.uid: 74882727-0808-43bb-ac3a-7b813af7c3ee
Type: kubernetes.io/service-account-token
Data
====
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IjBfd1JIa0ItdTZnaTZONUxFc192dTBFc2VWYjh3TV9zMmxIeU1zYWQtSUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLXRva2VuLWh4NWg4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6InByb21ldGhldXMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI3NDg4MjcyNy0wODA4LTQzYmItYWMzYS03YjgxM2FmN2MzZWUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06cHJvbWV0aGV1cyJ9.VTBej_PRKHRjMK4yI_JKm2dWb_s_ndN4NQ08k22Pl7yLilj62iZYoE0hywzpMLL149gHQmLyITmFODyJz98WfFeJS3h6RKsolNyBxE_3zvvKAqHG-RzI-LSrqBYFexfEilKwuQZ6K8cmjlJjxq1Gya3vE1MFeOT3d51tzV15hn-WtxNiOlEbwZno5hhfSLazS9seLjpnYrv02lUk-tZ5Fxv5E0XaEf6PbXRVYfn42d105_5wMvkA3lrqe3IK-u14awoKgH8MbqsDgqTCp0l8iePwc-s_zVL6FCeQSTnBZc0j9SWoUdIJIbAxhRbpwimmqeBomwFEGkSK-aGn82khJw
ca.crt: 1066 bytes
namespace: 11 bytes
新增prometheus配置项
[root@slave-2 prometheus]# cat k8s.token
eyJhbGciOiJSUzI1NiIsImtpZCI6IjBfd1JIa0ItdTZnaTZONUxFc192dTBFc2VWYjh3TV9zMmxIeU1zYWQtSUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJwcm9tZXRoZXVzLXRva2VuLWh4NWg4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6InByb21ldGhldXMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI3NDg4MjcyNy0wODA4LTQzYmItYWMzYS03YjgxM2FmN2MzZWUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06cHJvbWV0aGV1cyJ9.VTBej_PRKHRjMK4yI_JKm2dWb_s_ndN4NQ08k22Pl7yLilj62iZYoE0hywzpMLL149gHQmLyITmFODyJz98WfFeJS3h6RKsolNyBxE_3zvvKAqHG-RzI-LSrqBYFexfEilKwuQZ6K8cmjlJjxq1Gya3vE1MFeOT3d51tzV15hn-WtxNiOlEbwZno5hhfSLazS9seLjpnYrv02lUk-tZ5Fxv5E0XaEf6PbXRVYfn42d105_5wMvkA3lrqe3IK-u14awoKgH8MbqsDgqTCp0l8iePwc-s_zVL6FCeQSTnBZc0j9SWoUdIJIbAxhRbpwimmqeBomwFEGkSK-aGn82khJw
- job_name: kubernetes-nodes-cadvisor
metrics_path: /metrics
scheme: https
kubernetes_sd_configs:
- role: node
api_server: https://192.168.2.60:6443
bearer_token_file: /opt/monitor/prometheus/token.k8s
tls_config:
insecure_skip_verify: true
bearer_token_file: /opt/monitor/prometheus/token.k8s
tls_config:
insecure_skip_verify: true
relabel_configs:
# 将标签(.*)作为新标签名，原有值不变
- action: labelmap
regex: __meta_kubernetes_node_label_(.*)
# 修改NodeIP:10250为APIServerIP:6443
- action: replace
regex: (.*)
source_labels: ["__address__"]
target_label: __address__
replacement: 192.168.2.60:6443
# 实际访问指标接口 https://NodeIP:10250/metrics/cadvisor 这个接口只能APISERVER访问，故此重新标记标签使用APISERVER代理访问
- action: replace
source_labels: [__meta_kubernetes_node_name]
target_label: __metrics_path__
regex: (.*)
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
