prometheus 监控案例-kube-state-metrics

  kube-state-metrics 组件介绍

  github项目地址：https://github.com/kubernetes/kube-state-metrics

  Kube-state-metrics: 通过监听 API Server 生成有关资源对象的状态指标，比如 Deployment、Node、Pod，需要注意的是 kube-state-metrics 的使用场景不是用于监控对方是否存活，而是用于周期性获取目标对象的 metrics 指标数据并在 web 界面进行显示或被 prometheus 抓取(如 pod 的状态是 running 还是 Terminating、pod 的创建时间等) ，目前的 kube-state-metrics 收集的指标数据可参见官方的文档 ，https://github.com/kubernetes/kube-state-metrics/tree/master/docs ,并不会存储这些指标数据，所以可以使用 Prometheus 来抓取这些数据然后存储，主要关注的是业务相关的一些元数据，比如 Deployment、Pod、副本状态等，调度了多少个 replicas？现在可用的有几个？多少个 Pod 是 running/stopped/terminated 状态？Pod 重启了多少次？目前有多少 job 在运行中。

  官方的定义：

  kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects.

 

  镜像名称：

  

  将官方镜像替换镜像为阿里云镜像

  registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.6.0

  registry.cn-hangzhou.aliyuncs.com/liangxiaohui/kuberntes-state-metrics:v2.6.0

 

  部署kube-state-metric

root@master1:~/yaml# cat kube-state-metrics-deploy.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kube-state-metrics
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kube-state-metrics
  template:
    metadata:
      labels:
        app: kube-state-metrics
    spec:
      serviceAccountName: kube-state-metrics
      containers:
      - name: kube-state-metrics
        image: registry.cn-hangzhou.aliyuncs.com/liangxiaohui/kuberntes-state-metrics:v2.6.0 
        ports:
        - containerPort: 8080

---
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-state-metrics
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kube-state-metrics
rules:
- apiGroups: [""]
  resources: ["nodes", "pods", "services", "resourcequotas", "replicationcontrollers", "limitranges", "persistentvolumeclaims", "persistentvolumes", "namespaces", "endpoints"]
  verbs: ["list", "watch"]
- apiGroups: ["extensions"]
  resources: ["daemonsets", "deployments", "replicasets"]
  verbs: ["list", "watch"]
- apiGroups: ["apps"]
  resources: ["statefulsets"]
  verbs: ["list", "watch"]
- apiGroups: ["batch"]
  resources: ["cronjobs", "jobs"]
  verbs: ["list", "watch"]
- apiGroups: ["autoscaling"]
  resources: ["horizontalpodautoscalers"]
  verbs: ["list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kube-state-metrics
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-state-metrics
subjects:
- kind: ServiceAccount
  name: kube-state-metrics
  namespace: kube-system

---
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: 'true'
  name: kube-state-metrics
  namespace: kube-system
  labels:
    app: kube-state-metrics
spec:
  type: NodePort
  ports:
  - name: kube-state-metrics
    port: 8080
    targetPort: 8080
    nodePort: 31666
    protocol: TCP
  selector:
    app: kube-state-metrics

 

  验证状态

root@master1:~/yaml\# kubectl get pods -n kube-system |grep kube-state-metrics
kube-state-metrics-55c776795d-chvg8        1/1     Running   0              3m31s

  

  k8s内置prometheus-server配置采集指标

root@master1:~/yaml# vim prometheus-cfg.yml
  - job_name: "kube-state-metrics"
    static_configs:
      - targets: ["192.168.100.3:31666"]

root@master1:~/yaml# kubectl apply -f prometheus-cfg.yaml
root@master1:~/yaml# kubectl delete pods prometheus-server-84c8755ccc-tgwm6  #重启pod

  浏览器访问验证

  

  grafana导入dashboard模板

   ID：13824

  master和node节点数量的指标修改

count(node_load1{kubernetes_io_role="master"})
count(node_load1{kubernetes_io_role="node"})

 

 

   ID：14518