prometheus-k8s

一，prometheus在k8s中部署

1,创建命名空间

root@slave001:/apps# kubectl create ns monitoring
namespace/monitoring created

2,k8s每个节点部署cadvisor 8080

root@master001:~/prometheus# cat case1-daemonset-deploy-cadvisor.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: cadvisor
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: cAdvisor
  template:
    metadata:
      labels:
        app: cAdvisor
    spec:
      tolerations:    #污点容忍,忽略master的NoSchedule
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
      hostNetwork: true
      restartPolicy: Always   # 重启策略
      containers:
      - name: cadvisor
        image: gexuchuan123/cadvisor:v0.39.2 
        imagePullPolicy: IfNotPresent  # 镜像策略
        ports:
        - containerPort: 8080
        volumeMounts:
          - name: root
            mountPath: /rootfs
          - name: run
            mountPath: /var/run
          - name: sys
            mountPath: /sys
          - name: docker
            mountPath: /var/lib/docker
      volumes:
      - name: root
        hostPath:
          path: /
      - name: run
        hostPath:
          path: /var/run
      - name: sys
        hostPath:
          path: /sys
      - name: docker
        hostPath:
          path: /var/lib/docker

 

 

3,k8s每个节点部署node-exporter  9100

root@master001:~/prometheus# cat case2-daemonset-deploy-node-exporter.yaml 
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: node-exporter
  namespace: monitoring 
  labels:
    k8s-app: node-exporter
spec:
  selector:
    matchLabels:
        k8s-app: node-exporter
  template:
    metadata:
      labels:
        k8s-app: node-exporter
    spec:
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
      containers:
      - image: prom/node-exporter:v1.3.1 
        imagePullPolicy: IfNotPresent
        name: prometheus-node-exporter
        ports:
        - containerPort: 9100
          hostPort: 9100
          protocol: TCP
          name: metrics
        volumeMounts:
        - mountPath: /host/proc
          name: proc
        - mountPath: /host/sys
          name: sys
        - mountPath: /host
          name: rootfs
        args:
        - --path.procfs=/host/proc
        - --path.sysfs=/host/sys
        - --path.rootfs=/host
      volumes:
        - name: proc
          hostPath:
            path: /proc
        - name: sys
          hostPath:
            path: /sys
        - name: rootfs
          hostPath:
            path: /
      hostNetwork: true
      hostPID: true
---
apiVersion: v1
kind: Service
metadata:
  annotations:
    prometheus.io/scrape: "true"
  labels:
    k8s-app: node-exporter
  name: node-exporter
  namespace: monitoring 
spec:
  type: NodePort
  ports:
  - name: http
    port: 9100
    nodePort: 39100
    protocol: TCP
  selector:
    k8s-app: node-exporter

 

 

 

4,查看

root@slave001:/apps# kubectl get po -nmonitoring -owide
NAME                  READY   STATUS    RESTARTS   AGE     IP                NODE              NOMINATED NODE   READINESS GATES
cadvisor-f58s5        1/1     Running   0          20m     192.168.192.152   192.168.192.152   <none>           <none>
cadvisor-np42s        1/1     Running   0          20m     192.168.192.153   192.168.192.153   <none>           <none>
cadvisor-v4mwr        1/1     Running   0          20m     192.168.192.151   192.168.192.151   <none>           <none>
node-exporter-897xp   1/1     Running   5          4m55s   192.168.192.151   192.168.192.151   <none>           <none>
node-exporter-gvkm9   1/1     Running   5          4m55s   192.168.192.152   192.168.192.152   <none>           <none>
node-exporter-xx5vg   1/1     Running   5          4m55s   192.168.192.153   192.168.192.153   <none>           <none>

5,在k8s中部署prometheus监控数据

root@master001:~/prometheus# mkdir -p /data/prometheusdata   #创建挂载目录
root@master001:~/prometheus# chmod 777 /data/prometheusdata/ 

root@master001:~/prometheus# kubectl create serviceaccount monitor -nmonitoring  #创建监控账号

root@master001:~/prometheus# kubectl create clusterrolebinding monitor-clusterrolebinding -nmonitoring --clusterrole=cluster-admin --serviceaccount=monitoring:monitor
clusterrolebinding.rbac.authorization.k8s.io/monitor-clusterrolebinding created  #对monitoring账号授权

root@master001:~/prometheus# cat case3-1-prometheus-cfg.yaml    #prometheus配置到ConfigMap
---
kind: ConfigMap
apiVersion: v1
metadata:
  labels:
    app: prometheus
  name: prometheus-config
  namespace: monitoring 
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      scrape_timeout: 10s
      evaluation_interval: 1m
    scrape_configs:
    - job_name: 'kubernetes-node'
      kubernetes_sd_configs:
      - role: node
      relabel_configs:
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
        action: replace
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
    - job_name: 'kubernetes-node-cadvisor'
      kubernetes_sd_configs:
      - role:  node
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
    - job_name: 'kubernetes-apiserver'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https
    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

 

root@master001:~/prometheus# kubectl apply -f case3-1-prometheus-cfg.yaml 
configmap/prometheus-config created
root@master001:~/prometheus# kubectl get configmaps -nmonitoring
NAME                DATA   AGE
kube-root-ca.crt    1      91m
prometheus-config   1      22s

root@master001:~/prometheus# kubectl describe configmaps -nmonitoring prometheus-config
Name:         prometheus-config
Namespace:    monitoring
Labels:       app=prometheus
Annotations:  <none>

Data
====
prometheus.yml:
----
global:
  scrape_interval: 15s
  scrape_timeout: 10s
  evaluation_interval: 1m
scrape_configs:
- job_name: 'kubernetes-node'
  kubernetes_sd_configs:
  - role: node
  relabel_configs:
  - source_labels: [__address__]
    regex: '(.*):10250'
    replacement: '${1}:9100'
    target_label: __address__
    action: replace
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
- job_name: 'kubernetes-node-cadvisor'
  kubernetes_sd_configs:
  - role:  node
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  relabel_configs:
  - action: labelmap
    regex: __meta_kubernetes_node_label_(.+)
  - target_label: __address__
    replacement: kubernetes.default.svc:443
  - source_labels: [__meta_kubernetes_node_name]
    regex: (.+)
    target_label: __metrics_path__
    replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-apiserver'
  kubernetes_sd_configs:
  - role: endpoints
  scheme: https
  tls_config:
    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
  relabel_configs:
  - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
    action: keep
    regex: default;kubernetes;https
- job_name: 'kubernetes-service-endpoints'
  kubernetes_sd_configs:
  - role: endpoints
  relabel_configs:
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
    action: keep
    regex: true
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
    action: replace
    target_label: __scheme__
    regex: (https?)
  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
    action: replace
    target_label: __metrics_path__
    regex: (.+)
  - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
    action: replace
    target_label: __address__
    regex: ([^:]+)(?::\d+)?;(\d+)
    replacement: $1:$2
  - action: labelmap
    regex: __meta_kubernetes_service_label_(.+)
  - source_labels: [__meta_kubernetes_namespace]
    action: replace
    target_label: kubernetes_namespace
  - source_labels: [__meta_kubernetes_service_name]
    action: replace
    target_label: kubernetes_name

Events:  <none>

 

root@master001:~/prometheus# kubectl apply -f case3-2-prometheus-deployment.yaml   #deploy部署
deployment.apps/prometheus-server created
root@master001:~/prometheus# cat case3-2-prometheus-deployment.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: prometheus-server
  namespace: monitoring
  labels:
    app: prometheus
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
      component: server
    #matchExpressions:
    #- {key: app, operator: In, values: [prometheus]}
    #- {key: component, operator: In, values: [server]}
  template:
    metadata:
      labels:
        app: prometheus
        component: server
      annotations:
        prometheus.io/scrape: 'false'
    spec:
      nodeName: 192.168.192.151
      serviceAccountName: monitor
      containers:
      - name: prometheus
        image: prom/prometheus:v2.31.2
        imagePullPolicy: IfNotPresent
        command:
          - prometheus
          - --config.file=/etc/prometheus/prometheus.yml
          - --storage.tsdb.path=/prometheus
          - --storage.tsdb.retention=720h
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/prometheus/prometheus.yml
          name: prometheus-config
          subPath: prometheus.yml
        - mountPath: /prometheus/
          name: prometheus-storage-volume
      volumes:
        - name: prometheus-config
          configMap:
            name: prometheus-config
            items:
              - key: prometheus.yml
                path: prometheus.yml
                mode: 0644
        - name: prometheus-storage-volume
          hostPath:
           path: /data/prometheusdata
           type: Directory

 

root@master001:~/prometheus# kubectl apply -f case3-3-prometheus-svc.yaml   #svc部署
service/prometheus created
root@master001:~/prometheus# cat case3-3-prometheus-svc.yaml 
---
apiVersion: v1
kind: Service
metadata:
  name: prometheus
  namespace: monitoring
  labels:
    app: prometheus
spec:
  type: NodePort
  ports:
    - port: 9090
      targetPort: 9090
      nodePort: 30090
      protocol: TCP
  selector:
    app: prometheus
    component: server

 

 

root@master002:~# netstat -natpl|grep 10250
tcp        0      0 192.168.192.242:10250   0.0.0.0:*               LISTEN      946/kubelet         
tcp        0      0 192.168.192.242:10250   192.168.192.242:48770   ESTABLISHED 946/kubelet         
tcp        0      0 192.168.192.242:48770   192.168.192.242:10250   ESTABLISHED 869/kube-apiserver  
tcp        0      0 192.168.192.242:59380   192.168.192.243:10250   ESTABLISHED 869/kube-apiserver  
tcp        0      0 192.168.192.242:53984   192.168.192.241:10250   ESTABLISHED 869/kube-apiserver 

二，consul 集群部署

https://releases.hashicorp.com/consul/

环境    192.168.192.246-248    consul

            192.168.192.241-243   k8s

            192.168.192.182          prometheus

root@ubuntu20:/apps# unzip consul_1.11.0_linux_amd64.zip
Archive:  consul_1.11.0_linux_amd64.zip
  inflating: consul                  
root@ubuntu20:/apps# cp consul /usr/local/bin/

root@ubuntu20:/apps# consul -h  #验证
Usage: consul [--version] [--help] <command> [<args>]

mkdir /data/consul -p

root@ubuntu246:/apps# nohup consul agent -server -bootstrap -bind=192.168.192.246 -client=192.168.192.246 --data-dir=/data/consul -ui -node=192.168.192.246 &
root@ubuntu247:/apps# nohup consul agent  -bind=192.168.192.247 -client=192.168.192.247 --data-dir=/data/consul  -node=192.168.192.247 -join=192.168.192.246 &
nohup: ignoring input and appending output to 'nohup.out'
root@ubuntu20:/apps# nohup consul agent  -bind=192.168.192.248 -client=192.168.192.248 --data-dir=/data/consul  -node=192.168.192.248 -join=192.168.192.246 &
nohup: ignoring input and appending output to 'nohup.out'

 

root@jiajiedian:/apps# curl  http://192.168.192.246:8500/v1/agent/members
root@ubuntu20:/apps# curl -X PUT -d '{"id": "node-exporter-241","name": "node-exporter-241","address": "192.168.192.241","port": 9100,"tags": ["service"],"checks": [{"http": "http://192.168.192.241:9100/","interval": "5s"}]}' http://192.168.192.246:8500/v1/agent/service/register
root@ubuntu20:/apps# curl -X PUT -d '{"id": "node-exporter-242","name": "node-exporter-242","address": "192.168.192.242","port": 9100,"tags": ["service"],"checks": [{"http": "http://192.168.192.242:9100/","interval": "5s"}]}' http://192.168.192.246:8500/v1/agent/service/register
root@ubuntu20:/apps# curl -X PUT -d '{"id": "node-exporter-243","name": "node-exporter-243","address": "192.168.192.243","port": 9100,"tags": ["service"],"checks": [{"http": "http://192.168.192.243:9100/","interval": "5s"}]}' http://192.168.192.246:8500/v1/agent/service/register

root@shanchu:/apps# curl --request PUT http://192.168.192.246:8500/v1/agent/service/deregister/node-exporter-242

 

  - job_name: 'consul-prometheus'
    consul_sd_configs:
    - server: '192.168.192.246:8500'
      services: []  
    relabel_configs:
      - source_labels: [__meta_consul_service]
        regex: .*consul*
        action: drop

 三,file_sd_config

root@pro182:/apps/prometheus# cat prometheus.yml
global:
  scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 5s # Evaluate rules every 15 seconds. The default is every 1 minute.
#alerting:
#  alertmanagers:
#    - static_configs:
#        - targets:
#          - 192.168.192.189:9093
rule_files:
  - "/apps/prometheus/*.yaml"
scrape_configs:
  - job_name: "prometheus-cadvisor"
    static_configs:
      - targets: ["192.168.192.243:8080","192.168.192.242:8080","192.168.192.241:8080"]
  - job_name: 'file_sd_my_server'
    file_sd_configs:
      - files:
        - /apps/prometheus/file_sd/sd_my_server.json
        refresh_interval: 10s 

root@pro182:/apps/prometheus# cat  /apps/prometheus/file_sd/sd_my_server.json
[
  {
    "targets": [ "192.168.192.241:9100","192.168.192.243:9100"]  
  }
]

 

 

 

root@pro182:/apps/prometheus# cat prometheus.yml
global:
  scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 5s # Evaluate rules every 15 seconds. The default is every 1 minute.
#alerting:
#  alertmanagers:
#    - static_configs:
#        - targets:
#          - 192.168.192.189:9093
rule_files:
  - "/apps/prometheus/*.yaml"
scrape_configs:
  - job_name: "prometheus-cadvisor"
    static_configs:
      - targets: ["192.168.192.243:8080","192.168.192.242:8080","192.168.192.241:8080"]
  - job_name: 'file_sd_my_server'
    file_sd_configs:
      - files:
        - /apps/prometheus/file_sd/sd_my_server.json
        refresh_interval: 10s
  - job_name: 'kubernetes-nodes-monitor' 
    scheme: http 
    tls_config: 
      insecure_skip_verify: true 
    bearer_token_file: /apps/prometheus/k8s.token 
    kubernetes_sd_configs: 
    - role: node 
      api_server: https://192.168.192.241:6443 
      tls_config: 
        insecure_skip_verify: true 
      bearer_token_file: /apps/prometheus/k8s.token 
    relabel_configs: 
      - source_labels: [__address__] 
        regex: '(.*):10250' 
        replacement: '${1}:9100' 
        target_label: __address__ 
        action: replace 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: '${1}' 
        action: replace 
        target_label: LOC 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: 'NODE' 
        action: replace 
        target_label: Type 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: 'K3S-test' 
        action: replace 
        target_label: Env 
      - action: labelmap 
        regex: __meta_kubernetes_node_label_(.+) 


  - job_name: 'kubernetes-pods-monitor' 
    kubernetes_sd_configs: 
    - role: pod 
      api_server: https://192.168.192.241:6443 
      tls_config: 
        insecure_skip_verify: true 
      bearer_token_file: /apps/prometheus/k8s.token 
    tls_config:
      insecure_skip_verify: true
    bearer_token_file: /apps/prometheus/k8s.token
    relabel_configs: 
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] 
      action: keep 
      regex: true 
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] 
      action: replace 
      target_label: __metrics_path__ 
      regex: (.+) 
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] 
      action: replace 
      regex: ([^:]+)(?::\d+)?;(\d+) 
      replacement: $1:$2 
      target_label: __address__ 
    - action: labelmap 
      regex: __meta_kubernetes_pod_label_(.+) 
    - source_labels: [__meta_kubernetes_namespace] 
      action: replace 
      target_label: kubernetes_namespace 
    - source_labels: [__meta_kubernetes_pod_name] 
      action: replace 
      target_label: kubernetes_pod_name 
    - source_labels: [__meta_kubernetes_pod_label_pod_template_hash] 
      regex: '(.*)' 
      replacement: 'K8S-test' 
      action: replace 
      target_label: Env

 四，在k8s外部授权promethes

root@master001:~/prometheus# cat case4-prom-rbac.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - services
  - endpoints
  - pods
  - nodes/proxy
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - "extensions"
  resources:
    - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - configmaps
  - nodes/metrics
  verbs:
  - get
- nonResourceURLs:
  - /metrics
  verbs:
  - get
---
#apiVersion: rbac.authorization.k8s.io/v1beta1
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: monitoring

root@master001:~/prometheus# kubectl get serviceaccount  -nmonitoring
NAME         SECRETS   AGE
default      1         2d23h
monitor      1         2d22h
prometheus   1         72s

root@master001:~/prometheus# kubectl get secret -nmonitoring
NAME                     TYPE                                  DATA   AGE
default-token-r9hd5      kubernetes.io/service-account-token   3      2d23h
monitor-token-kbvpj      kubernetes.io/service-account-token   3      2d22h
prometheus-token-srhnl   kubernetes.io/service-account-token   3      2m8s

root@master001:~/prometheus# kubectl describe secret prometheus-token-srhnl -nmonitoring
Name:         prometheus-token-srhnl
Namespace:    monitoring
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: prometheus
              kubernetes.io/service-account.uid: 2d9853d0-adfe-415c-963d-dd1da807c6cc

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1350 bytes
namespace:  10 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6Ikc1YlhjN0w1SkJhcmRIWkdwRnBzMlM2NFh4dWxWZ0hkVXVwU0I3dWtFWEEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJtb25pdG9yaW5nIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6InByb21ldGhldXMtdG9rZW4tc3JobmwiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoicHJvbWV0aGV1cyIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjJkOTg1M2QwLWFkZmUtNDE1Yy05NjNkLWRkMWRhODA3YzZjYyIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDptb25pdG9yaW5nOnByb21ldGhldXMifQ.r7KzJqb1cDRiC_ioFuHeysQpzmOYTJr_47acfxQq4ngczhv3sn7O0_evOLRFWdfwcG-S80h93Us1rKPu-5tJYdYpo66rWYkQpF8EgjtWdd7UPVD2VG56tm249oL9BFNaRCayEMXLZycy_hX39hR4UcZnfhkHvsnxTiIjRI_S-sFEMFiy9VhD6Szq3VNwDqj197CflaiQqhTj0vVBSZiD16EuTsRE6T_sZDUIByXQIbKmuj2OvaqWGd5zidPeDLcXCbtTOr_TQx8gXFRwL1ZkQnjVp71nvRR8DuRv6f9CwCr4dVnQD1gkbXOISWaGuDA0vtEUdj8s1LW740XO8SraCg

root@pro182:/apps/prometheus# cat prometheus.yml
# my global config
global:
  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
  # scrape_timeout is set to the global default (10s).

# Alertmanager configuration
alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - 192.168.192.189:9093

# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
rule_files:
  - "/apps/prometheus/*.yaml"
  # - "first_rules.yml"
  # - "second_rules.yml"

scrape_configs:
  - job_name: "prometheus"
    static_configs:
      - targets: ["localhost:9090"]
  - job_name: "prometheus-node_exporter"
    static_configs:
      - targets: ["192.168.192.151:39100","192.168.192.152:39100","192.168.192.153:39100"]
#网站监控
  - job_name: 'wangzhanjiankong-9115'
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets: ['http://www.baidu.com', 'http://www.xiaomi.com']
        labels:
          instance: http_status
          group: web
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: url
      - target_label: __address__
        replacement: 192.168.192.189:9115

#端口监控
  - job_name: 'port_6443'
    metrics_path: /probe
    params:
      module: [tcp_connect]
    static_configs:
      - targets: ['192.168.192.151:6443', '192.168.192.152:6443']
        labels:
          instance: port_status
          group: port
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: url
      - target_label: __address__
        replacement: 192.168.192.189:9115
  - job_name: "prometheus-cadvisor"
    static_configs:
      - targets: ["192.168.192.151:8080","192.168.192.152:8080","192.168.192.153:8080"]

  #二进制监控k8s
  - job_name: "k8s-api-monitor"
    kubernetes_sd_configs:
    - role: endpoints
      api_server: https://192.168.192.151:6443
      tls_config:
        insecure_skip_verify: true
      bearer_token_file: /apps/prometheus/k8s.token
    scheme: https
    tls_config:
      insecure_skip_verify: true
    bearer_token_file: /apps/prometheus/k8s.token
    relabel_configs:
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: default;kubernetes;https
    - target_label: __address__
      replacement: 192.168.192.151:6443

 

 

- job_name: 'kubernetes-nodes-monitoring' 
    scheme: http 
    tls_config: 
      insecure_skip_verify: true 
    bearer_token_file: /apps/prometheus/k8s.token 
    kubernetes_sd_configs: 
    - role: node 
      api_server: https://192.168.192.151:6443 
      tls_config: 
        insecure_skip_verify: true 
      bearer_token_file: /apps/prometheus/k8s.token 
    relabel_configs: 
      - source_labels: [__address__] 
        regex: '(.*):10250' 
        replacement: '${1}:9100' 
        target_label: __address__ 
        action: replace 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: '${1}' 
        action: replace 
        target_label: LOC 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: 'NODE' 
        action: replace 
        target_label: Type 
      - source_labels: [__meta_kubernetes_node_label_failure_domain_beta_kubernetes_io_region] 
        regex: '(.*)' 
        replacement: 'K8S-test' 
        action: replace 
        target_label: Env 
      - action: labelmap 
        regex: __meta_kubernetes_node_label_(.+)