Prometheus监控携带metrics接口的服务
Prometheus监控携带metrics接口的服务
prometheus监控分为两种：
1、携带metircs接口的服务
2、不携带metrics接口的服务
一、prometheus监控携带metrics接口的服务的流程：
携带metrics接口的服务就表示可以通过metrics接口获取服务的监控项和监控信息
（PS：此处以ETCD为例）
1、通过endprints获取锁需要监控的ETCD的地址
2、创建Service，给予集群内部的ServiceMoniter使用
3、创建ServiceMoniter部署需要的访问证书
4、重启promutheus监控Pod，载入监控项
二、通过prometheus监控ETCD
ETCD服务携带metrics接口，prometheus监控它的流程为：
1、通过EndPrints获取所要监控的ETCD的地址
2、创建Service，给予集群内部的ServiceMoniter使用
3、创建ServiceMoniter部署需要访问证书，给予prometheus-k8s-0来使用
4、重启prometheus监控Pod (prometheus-k8s-0)，载入监控项
1、测试ETCD服务的metrics接口
[root@k8s-master-01 ~]# netstat -lnutp | grep etcd
tcp 0 0 192.168.15.31:2379 0.0.0.0:* LISTEN 2692/etcd
tcp 0 0 127.0.0.1:2379 0.0.0.0:* LISTEN 2692/etcd
tcp 0 0 192.168.15.31:2380 0.0.0.0:* LISTEN 2692/etcd
tcp 0 0 127.0.0.1:2381 0.0.0.0:* LISTEN 2692/etcd
[root@k8s-master-01 ~]# curl -k --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key https://127.0.0.1:2379/metrics
#ETCD端口：
2379 提供HTTP API服务
2380 和集群中其他节点通信
2、通过EndPrints获取所需监控的ETCD的地址
kind: Endpoints
apiVersion: v1
metadata:
namespace: kube-system
name: etcd-moniter
labels:
k8s: etcd
subsets:
- addresses:
- ip: "192.168.15.31"
ports:
- port: 2379
protocol: TCP
name: etcd
[root@k8s-master-01 ~]# kubectl apply -f endprints.yaml
endpoints/etcd-moniter created
[root@k8s-master-01 ~]# kubectl get endpoints -n kube-system
NAME ENDPOINTS AGE
etcd-moniter 192.168.15.31:2379 17s
3、创建Service，给予集群内部的ServiceMoniter使用
kind: Service
apiVersion: v1
metadata:
name: etcd-moniter
namespace: kube-system
labels:
k8s: etcd
spec:
ports:
- port: 2379
targetPort: 2379
name: etcd
protocol: TCP
[root@k8s-master-01 ~]# kubectl apply -f service.yaml
service/etcd-moniter created
[root@k8s-master-01 ~]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
etcd-moniter ClusterIP 10.103.237.209 <none> 2379/TCP 13s
kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 25d
kubelet ClusterIP None <none> 10250/TCP,10255/TCP,4194/TCP 44h
[root@k8s-master-01 ~]# kubectl describe -n kube-system svc etcd-moniter
Name: etcd-moniter
Namespace: kube-system
Labels: k8s=etcd
Annotations: <none>
Selector: <none>
Type: ClusterIP
IP Families: <none>
IP: 10.103.237.209
IPs: 10.103.237.209
Port: etcd 2379/TCP
TargetPort: 2379/TCP
Endpoints: 192.168.15.31:2379
Session Affinity: None
Events: <none>
4、创建ServiceMoniter部署需要访问证书
kind: ServiceMonitor
apiVersion: monitoring.coreos.com/v1
metadata:
labels:
k8s: etcd
name: etcd-monitor
namespace: monitoring
spec:
endpoints:
- interval: 3s
port: etcd
scheme: https
tlsConfig:
caFile: /etc/prometheus/secrets/etcd-certs/ca.crt
certFile: /etc/prometheus/secrets/etcd-certs/peer.crt
keyFile: /etc/prometheus/secrets/etcd-certs/peer.key
insecureSkipVerify: true
selector:
matchLabels:
k8s: etcd
namespaceSelector:
matchNames:
- "kube-system"
[root@k8s-master-01 ~]# kubectl apply -f servicemoniter.yaml
servicemonitor.monitoring.coreos.com/etcd-monitor created
[root@k8s-master-01 ~]# kubectl get ServiceMonitor -n monitoring
NAME AGE
etcd-monitor 24s
5、重启prometheus监控Pod(prometheus-k8s-0)，载入监控项
创建一个secrets，用来保存prometheus监控的etcd的证书
[root@k8s-master-01 ~]# kubectl create secret generic etcd-certs -n monitoring --from-file=/etc/kubernetes/pki/etcd/ca.crt --from-file=/etc/kubernetes/pki/etcd/peer.crt --from-file=/etc/kubernetes/pki/etcd/peer.key
secret/etcd-certs created
[root@k8s-master-01 ~]# kubectl get secret -n monitoring
NAME TYPE DATA AGE
etcd-certs Opaque 3 14s
kind: Prometheus
apiVersion: monitoring.coreos.com/v1
metadata:
labels:
prometheus: k8s
name: k8s
namespace: monitoring
spec:
alerting:
alertmanagers:
- name: alertmanager-main
namespace: monitoring
port: web
- name: alertmanager-main-etcd
namespace: kube-system
port: etcd
image: quay.io/prometheus/prometheus:v2.15.2
nodeSelector:
kubernetes.io/os: linux
podMonitorNamespaceSelector: {}
podMonitorSelector: {}
replicas: 2
resources:
requests:
memory: 400Mi
ruleSelector:
matchLabels:
prometheus: k8s
role: alert-rules
securityContext:
fsGroup: 2000
runAsNonRoot: true
runAsUser: 1000
serviceAccountName: prometheus-k8s
serviceMonitorNamespaceSelector: {}
serviceMonitorSelector: {}
version: v2.15.2
secrets:
- etcd-certs
结果
[root@k8s-master-01 ~]# kubectl apply -f prometheus-k8s-0.yaml
prometheus.monitoring.coreos.com/k8s configured
[root@k8s-master-01 ~]# kubectl get pod -n monitoring
NAME READY STATUS RESTARTS AGE
prometheus-k8s-0 3/3 Running 0 24s
prometheus-k8s-1 0/3 Pending 0 24s
6、检测是否监控成功
补充：grafana图形美化
1、去grafana官网寻找合适个模板
2、使用模板
3、模板使用成功，看上去很装逼
