kubernetes监控和性能分析工具：heapster+influxdb+grafana

1.部署heapster

下载 heapster 相关 yaml 文件

[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/grafana.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/influxdb.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/influxdb/heapster.yaml
[root@master dashboard]# wget https://raw.githubusercontent.com/kubernetes-retired/heapster/master/deploy/kube-config/rbac/heapster-rbac.yaml
[root@master dashboard]# cat  heapster.yaml | grep image
        image: k8s.gcr.io/heapster-amd64:v1.5.4
        imagePullPolicy: IfNotPresent
[root@master dashboard]#  cat  grafana.yaml | grep image
        image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# cat influxdb.yaml | grep image
        image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2

 

1.1在node上pull和tag被墙的镜像

[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-amd64:v1.5.4
[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# docker pull mirrorgooglecontainers/heapster-influxdb-amd64:v1.5.2

[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-amd64:v1.5.4 k8s.gcr.io/heapster-amd64:v1.5.4
[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-grafana-amd64:v5.0.4 k8s.gcr.io/heapster-grafana-amd64:v5.0.4
[root@master dashboard]# docker tag mirrorgooglecontainers/heapster-influxdb-amd64:v1.5.2 k8s.gcr.io/heapster-influxdb-amd64:v1.5.2
[root@master dashboard]# ls heapster/
grafana.yaml  heapster-rbac.yaml  heapster.yaml  influxdb.yaml
[root@master dashboard]# kubectl apply -f ./heapster/
deployment.extensions/monitoring-grafana created
service/monitoring-grafana created
clusterrolebinding.rbac.authorization.k8s.io/heapster created
serviceaccount/heapster created
deployment.extensions/heapster created
service/heapster created
deployment.extensions/monitoring-influxdb created
service/monitoring-influxdb created
[root@master dashboard]# kubectl get pods -n kube-system -owide
NAME                                    READY   STATUS    RESTARTS   AGE   IP            NODE     NOMINATED NODE   READINESS GATES
coredns-fb8b8dccf-bh4t2                 1/1     Running   0          14d   10.244.0.3    master   <none>           <none>
coredns-fb8b8dccf-rxkjw                 1/1     Running   0          14d   10.244.0.2    master   <none>           <none>
etcd-master                             1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
heapster-5d4bf58946-bfvcz               1/1     Running   0          13m   10.244.2.28   node01   <none>           <none>
kube-apiserver-master                   1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
kube-controller-manager-master          1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
kube-flannel-ds-amd64-82884             1/1     Running   0          14d   10.0.1.229    node01   <none>           <none>
kube-flannel-ds-amd64-nhvm5             1/1     Running   2          14d   10.0.1.7      node02   <none>           <none>
kube-flannel-ds-amd64-xwng8             1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
kube-proxy-2mk2h                        1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
kube-proxy-54x8x                        1/1     Running   0          14d   10.0.1.7      node02   <none>           <none>
kube-proxy-n9rdx                        1/1     Running   0          14d   10.0.1.229    node01   <none>           <none>
kube-scheduler-master                   1/1     Running   0          14d   10.0.1.195    master   <none>           <none>
kubernetes-dashboard-769b74d4b4-kk6zx   1/1     Running   0          38m   10.244.2.27   node01   <none>           <none>
monitoring-grafana-658976d65f-xgp6v     1/1     Running   0          13m   10.244.1.18   node02   <none>           <none>
monitoring-influxdb-866db5f944-8tq74    1/1     Running   0          13m   10.244.1.19   node02   <none>           <none>
tiller-deploy-58f5d95b9c-x492l          1/1     Running   0          13d   10.244.2.3    node01   <none>           <none>
traefik-ingress-controller-f9bg6        1/1     Running   0          73m   10.244.1.16   node02   <none>           <none>
traefik-ingress-controller-x6tjl        1/1     Running   0          73m   10.244.2.26   node01   <none>           <none>
[root@master dashboard]# kubectl logs  heapster-5d4bf58946-bfvcz -n kube-system
E0614 16:01:05.004107       1 manager.go:101] Error in scraping containers from kubelet:10.0.1.195:10255: failed to get all container stats from Kubelet URL "http://10.0.1.195:10255/stats/container/": Post http://10.0.1.195:10255/stats/container/: dial tcp 10.0.1.195:10255: getsockopt: connection refused
W0614 16:01:25.000648       1 manager.go:152] Failed to get all responses in time (got 0/3)
E0614 16:05:05.022570       1 manager.go:101] Error in scraping containers from kubelet:10.0.1.229:10255: failed to get all container stats from Kubelet URL "http://10.0.1.229:10255/stats/container/": Post http://10.0.1.229:10255/stats/container/: dial tcp 10.0.1.229:10255: getsockopt: connection refused
W0614 16:05:25.000593       1 manager.go:152] Failed to get all responses in time (got 0/3)

//上面还有报错

1.2修改heapster.yaml 文件

#### 修改如下部分 #####
因为 kubelet 启用了 https 所以如下配置需要增加 https 端口
        - --source=kubernetes:https://kubernetes.default
修改为
        - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true

1.3修改heapster-rbac.yaml文件

#### 修改为部分 #####
将 serviceAccount kube-system:heapster 与 ClusterRole system:kubelet-api-admin 绑定，授予它调用 kubelet API 的权限；
[root@master heapster]# cat heapster-rbac.yaml 
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster-kubelet-api
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system

2创建应用

[root@master heapster]# kubectl apply -f ./
[root@master heapster]# kubectl logs  heapster-5d9575b66b-t5zxt -n kube-system
I0614 16:16:09.353325       1 heapster.go:78] /heapster --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
I0614 16:16:09.353365       1 heapster.go:79] Heapster version v1.5.4
I0614 16:16:09.353623       1 configs.go:61] Using Kubernetes client with master "https://kubernetes.default" and version v1
I0614 16:16:09.353648       1 configs.go:62] Using kubelet port 10250
I0614 16:16:09.367340       1 influxdb.go:312] created influxdb sink with options: host:monitoring-influxdb.kube-system.svc:8086 user:root db:k8s
I0614 16:16:09.367368       1 heapster.go:202] Starting with InfluxDB Sink
I0614 16:16:09.367374       1 heapster.go:202] Starting with Metric Sink
I0614 16:16:09.375501       1 heapster.go:112] Starting heapster on port 8082
I0614 16:17:05.067815       1 influxdb.go:274] Created database "k8s" on influxDB server at "monitoring-influxdb.kube-system.svc:8086"

3.查看监控节点

[root@master heapster]# kubectl top nodes
NAME     CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master   112m         1%     2464Mi          16%       
node01   59m          0%     4356Mi          13%       
node02   47m          0%     1644Mi          10%

4.访问

可以从kuberbetes-dashboard访问https://www.cnblogs.com/linyouyi/p/11001213.html

也可以把上面的应用单独暴露NodePort然后通过NodeIP:NodePort访问 

更加详细玩转https://www.cnblogs.com/scode2/p/8810052.html

 

grafana.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-grafana
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: grafana
    spec:
      containers:
      - name: grafana
        image: k8s.gcr.io/heapster-grafana-amd64:v5.0.4
        ports:
        - containerPort: 3000
          protocol: TCP
        volumeMounts:
        - mountPath: /etc/ssl/certs
          name: ca-certificates
          readOnly: true
        - mountPath: /var
          name: grafana-storage
        env:
        - name: INFLUXDB_HOST
          value: monitoring-influxdb
        - name: GF_SERVER_HTTP_PORT
          value: "3000"
          # The following env variables are required to make Grafana accessible via
          # the kubernetes api-server proxy. On production clusters, we recommend
          # removing these env variables, setup auth for grafana, and expose the grafana
          # service using a LoadBalancer or a public IP.
        - name: GF_AUTH_BASIC_ENABLED
          value: "false"
        - name: GF_AUTH_ANONYMOUS_ENABLED
          value: "true"
        - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          value: Admin
        - name: GF_SERVER_ROOT_URL
          # If you're only using the API Server proxy, set this value instead:
          # value: /api/v1/namespaces/kube-system/services/monitoring-grafana/proxy
          value: /
      volumes:
      - name: ca-certificates
        hostPath:
          path: /etc/ssl/certs
      - name: grafana-storage
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-grafana
  name: monitoring-grafana
  namespace: kube-system
spec:
  # In a production setup, we recommend accessing Grafana through an external Loadbalancer
  # or through a public IP.
  # type: LoadBalancer
  # You could also use NodePort to expose the service at a randomly-generated port
  # type: NodePort
  ports:
  - port: 80
    targetPort: 3000
  selector:
    k8s-app: grafana
View Code

 heapster-rbac.yaml

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:heapster
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: heapster-kubelet-api
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kubelet-api-admin
subjects:
- kind: ServiceAccount
  name: heapster
  namespace: kube-system
View Code

heapster.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: heapster
  namespace: kube-system
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: heapster
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: heapster
    spec:
      serviceAccountName: heapster
      containers:
      - name: heapster
        image: k8s.gcr.io/heapster-amd64:v1.5.4
        imagePullPolicy: IfNotPresent
        command:
        - /heapster
        - --source=kubernetes:https://kubernetes.default?kubeletHttps=true&kubeletPort=10250&insecure=true
        - --sink=influxdb:http://monitoring-influxdb.kube-system.svc:8086
---
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: Heapster
  name: heapster
  namespace: kube-system
spec:
  ports:
  - port: 80
    targetPort: 8082
  selector:
    k8s-app: heapster
View Code

influxdb.yaml

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: monitoring-influxdb
  namespace: kube-system
spec:
  replicas: 1
  template:
    metadata:
      labels:
        task: monitoring
        k8s-app: influxdb
    spec:
      containers:
      - name: influxdb
        image: k8s.gcr.io/heapster-influxdb-amd64:v1.5.2
        volumeMounts:
        - mountPath: /data
          name: influxdb-storage
      volumes:
      - name: influxdb-storage
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  labels:
    task: monitoring
    # For use as a Cluster add-on (https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
    # If you are NOT using this as an addon, you should comment out this line.
    kubernetes.io/cluster-service: 'true'
    kubernetes.io/name: monitoring-influxdb
  name: monitoring-influxdb
  namespace: kube-system
spec:
  ports:
  - port: 8086
    targetPort: 8086
  selector:
    k8s-app: influxdb
View Code

 

posted @ 2019-06-30 20:37  linyouyi  阅读(1136)  评论(0)  编辑  收藏  举报