prometheus学习笔记之集群内服务发现环境准备

一、环境介绍

主要演示prometheus在k8s集群中如何通过服务自动去发现k8s集群自有服务及其他服务发现场景，后续会演示集群外部署prometheus自动发现k8s服务并获取数据

创建监控使用的namespaces

kubectl create ns monitoring

配置docker可以下载镜像

[root@k8s-master deploy]# cat /etc/docker/daemon.json #配置registry-mirrors
{
  "registry-mirrors": ["https://docker.m.daocloud.io","https://huecker.io","https://dockerhub.timeweb.cloud"],
  "exec-opts": ["native.cgroupdriver=systemd"]
}
[

prometheus相关镜像参考地址：https://hub.docker.com/u/prom 都在prom下

二、k8s安装node-exporter

参考地址：https://github.com/prometheus/node_exporter

kubectl create ns monitoring #创建专有的NS
mkdir -p manifest/monitor
cd manifest/monitor

vim node-export-ds.yaml #注意类型是DaemonSet,相比二进制部署，daemonSet方式部署在节点经常变动的情况下避免了手动安装的问题
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitoring 
  labels:
    k8s-app: node-exporter
spec:
  selector:
    matchLabels:
        k8s-app: node-exporter
  template:
    metadata:
      labels:
        k8s-app: node-exporter
    spec:
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master #注意，你的master节点污点和我的未必一致 通过describe命令查看
      containers:
      - image: registry-vpc.cn-shanghai.aliyuncs.com/zdbl-base/node-exporter:v1.3.1 #这里我是下载到我的私有仓库，可以直接使用官方的镜像地址，docker配置后应当可以直接下载
        name: prometheus-node-exporter
        ports:
        - containerPort: 9100
          hostPort: 9100 #如果不写默认与containerPort一致，注意端口冲突
          protocol: TCP
          name: metrics
        volumeMounts:
        - mountPath: /host/proc
          name: proc
        - mountPath: /host/sys
          name: sys
        - mountPath: /host
          name: rootfs
        args:
        - --path.procfs=/host/proc
        - --path.sysfs=/host/sys
        - --path.rootfs=/host
      volumes:
        - name: proc
          hostPath:
            path: /proc
        - name: sys
          hostPath:
            path: /sys
        - name: rootfs
          hostPath:
            path: /
      hostNetwork: true
      hostPID: true
      hostIPC: true


kubectl apply -f node-export-ds.yaml
kubectl get pods -n monitoring 
NAME                  READY   STATUS    RESTARTS   AGE
node-exporter-7bp55   1/1     Running   0          1m
node-exporter-klx2b   1/1     Running   0          1m
node-exporter-pcht8   1/1     Running   0          1m
netstat -tnlp|grep 9100 #因为是hostPort

浏览器访问节点 9100 端口 /metrics 接口

二、k8s安装cadvisor

略，参考https://www.cnblogs.com/panwenbin-logs/p/18385045

三、安装NFS 存储类

如果有请忽略

参考文档：https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/

wget https://github.com/kubernetes-sigs/nfs-subdir-external-provisioner/archive/refs/tags/nfs-subdir-external-provisioner-4.0.18.tar.gz
tar xf nfs-subdir-external-provisioner-4.0.18.tar.gz
cd nfs-subdir-external-provisioner-nfs-subdir-external-provisioner-4.0.18/
cp -r deploy deploy-bak
cd deploy
kubectl create ns nfs-provisioner #创建专用的名称空间
sed -i 's/namespace: default/namespace: nfs-provisioner/g' `grep -rl 'namespace: default' ./` #替换配置文件名称空间
vim deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  # replace with namespace where provisioner is deployed
  namespace: nfs-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: registry-vpc.cn-shanghai.aliyuncs.com/zdbl-base/nfs-subdir-external-provisioner:v4.0.2
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: k8s-sigs.io/nfs-subdir-external-provisioner
            - name: NFS_SERVER  #server及挂载点根据你的实际情况进行配置，此处使用的是阿里云的NAS
              value: 332fa4bbd2-rho88.cn-shanghai.nas.aliyuncs.com
            - name: NFS_PATH
              value: /k8s-nfs-sc
      volumes:
        - name: nfs-client-root
          nfs:
            server: 332fa4bbd2-rho88.cn-shanghai.nas.aliyuncs.com #上面的保持一致
            path: /k8s-nfs-sc
kubectl apply -k .
kubectl get pods -n nfs-provisioner

验证

vim test-claim.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: test-claim
spec:
  storageClassName: nfs-client #注意此处名称和class.yaml中定义的名称保持一致
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 1Mi
---
kind: Pod
apiVersion: v1
metadata:
  name: test-pod
spec:
  containers:
  - name: test-pod
    image: busybox:stable
    command:
      - "/bin/sh"
    args:
      - "-c"
      - "touch /mnt/SUCCESS && exit 0 || exit 1" #挂载并创建一个文件
    volumeMounts:
      - name: nfs-pvc
        mountPath: "/mnt"
  restartPolicy: "Never"
  volumes:
    - name: nfs-pvc
      persistentVolumeClaim:
        claimName: test-claim

kubectl apply -f  test-pod.yaml

kubectl get pods
NAME READY STATUS RESTARTS AGE
test-pod 0/1 Completed 0 63m

kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pvc-f4e9b50d-6a19-4c7f-bdc9-18855c712fdd 1Mi RWX Delete Bound default/test-claim nfs-client 63m

kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
test-claim Bound pvc-f4e9b50d-6a19-4c7f-bdc9-18855c712fdd 1Mi RWX nfs-client 63m

ll /mnt/k8s-nfs-sc/default-test-claim-pvc-f4e9b50d-6a19-4c7f-bdc9-18855c712fdd/SUCCESS
-rw-r--r-- 1 root root 0 2024-09-03 09:29 /mnt/k8s-nfs-sc/default-test-claim-pvc-f4e9b50d-6a19-4c7f-bdc9-18855c712fdd/SUCCESS

设置为NFS默认SC

kubectl get sc
NAME         PROVISIONER                                   RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client   k8s-sigs.io/nfs-subdir-external-provisioner   Delete          Immediate           false                  14m

kubectl patch storageclass nfs-client -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'  #通过打补丁的方式设置 nfs-client 这个SC为默认的SC
storageclass.storage.k8s.io/nfs-client patched

kubectl get sc #可以看到nfs-clinet SC已经有default标识了
NAME                   PROVISIONER                                   RECLAIMPOLICY   VOLUMEBINDINGMODE   ALLOWVOLUMEEXPANSION   AGE
nfs-client (default)   k8s-sigs.io/nfs-subdir-external-provisioner   Delete          Immediate           false                  15m

四、k8s部署prometheus

1.创建prometheus相关配置文件

prometheus-cm.yaml

kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    app: prometheus
  name: prometheus-config
  namespace: monitoring
data:
  prometheus.yml: |
    global:  #全局配置，同二进制
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 1m
    scrape_configs: #数据抓取配置
    - job_name: 'kubernetes-node' #k8s自动发现
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'   #kubelet 默认访问10250 但是我们的node-export部署的端口是9100，所以在抓取数据前需要修改数据采集的端口
        replacement: '${1}:9100' #如果监控cadvisor 复制kubernetes-node job_name 将9100端口修改为cadvisor端口即可
        target_label: __address__
        action: replace
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    - job_name: 'kubernetes-node-cadvisor'  #这来抓取cadvisor数据通过 kubeapi，因为k8s已经内置了cadvisor，如果通过部署的DS获取，参考上面的 kubernetes-node job_name
      kubernetes_sd_configs:
      - role:  node
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt  #访问api-server需要认证，此处使用证书，相关证书和token已经自动挂载到pod中
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor #变量替换，将$1替换为node名称，名称从上面的regex 正则中获取
    - job_name: 'kubernetes-apiserver'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt #xxxxxxxxxxxx
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep #保留source_labels中标签，其他过滤
        regex: default;kubernetes;https   #与source_labels中对应，分别为名称空间  svc名称  svc中port的名称，注意此处是名称不是协议，协议在scheme字段配置
    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+) #匹配svc metadata中 lable的标签名称和值
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

prometheus-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: monitoring
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      component: server
  template:
    metadata:
      labels:
        app: prometheus
        component: server
      annotations:
        prometheus.io/scrape: 'false'
    spec:
      serviceAccountName: monitor
      containers:
      - name: prometheus
        image: registry-vpc.cn-shanghai.aliyuncs.com/zdbl-base/prometheus:v2.36.1
        command:
          - prometheus
          - --config.file=/etc/prometheus/prometheus.yml
          - --storage.tsdb.path=/prometheus
          - --storage.tsdb.retention=720h
          - --web.enable-lifecycle
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/prometheus/prometheus.yml
          name: prometheus-config
          subPath: prometheus.yml
        - mountPath: /prometheus/
          name: prometheus-storage-volume
      volumes:
        - name: prometheus-config
          configMap:
            name: prometheus-config
            items:
              - key: prometheus.yml
                path: prometheus.yml
                mode: 0644
        - name: prometheus-storage-volume  #这里使用了NFS存储类创建的PVC，如果没有可以直接使用hostPath方式
          persistentVolumeClaim:
            claimName: prometheus-data

prometheus-pvc.yaml

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
  name: prometheus-data
  namespace: monitoring
spec:
  storageClassName: nfs-client
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 10G

prometheus-rbac.yaml

apiVersion: v1
kind: ServiceAccount  #prometheus监控集群需要很多权限，此处为了方便支持通过 cluster-admin授权，实际应当给予所以资源读取权限即可
metadata:
  name: monitor
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: monitor-clusterrolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: monitor
  namespace: monitoring

prometheus-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitoring
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
    - port: 9090
      targetPort: 9090
      nodePort: 30090
      protocol: TCP
  selector:
    app: prometheus
    component: server

2.应用配置文件

kubectl apply -f .
kubectl get pods -n monitoring |grep prometheus
kubectl logs -n monitoring prometheus-server-5d5cc898b6-q69wp

3.访问web页面 节点IP:30090

 

prometheus参考：https://prometheus.io/docs/prometheus/latest/configuration/configuration/