续(基于外部prometheus监控k8s 集群及k8s应用服务 )
续：https://www.cnblogs.com/cyh00001/p/16725312.html
七、监控mysql服务
7.1、安装mysql服务
apt-get install mysql-server -y
systemctl enable --now mysql
7.2、创建监控账号并授权mysql
[root@easzlab-images-02 ~]# mysql Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 9 Server version: 8.0.30-0ubuntu0.20.04.2 (Ubuntu) Copyright (c) 2000, 2022, Oracle and/or its affiliates. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective owners. Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. mysql> mysql> CREATE USER 'mysql_exporter'@'localhost' IDENTIFIED BY 'imnot007*'; Query OK, 0 rows affected (0.01 sec) mysql> GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO 'mysql_exporter'@'localhost'; Query OK, 0 rows affected (0.02 sec) mysql> mysql> select User,Host from mysql.user; +------------------+-------------+ | User | Host | +------------------+-------------+ | mysql_exporter | localhost | | debian-sys-maint | localhost | | mysql.infoschema | localhost | | mysql.session | localhost | | mysql.sys | localhost | | root | localhost | +------------------+-------------+ 6 rows in set (0.00 sec) mysql>
7.3、安装部署mysql_exporter
wget https://github.com/prometheus/mysqld_exporter/releases/download/v0.14.0/mysqld_exporter-0.14.0.linux-amd64.tar.gz
tar -xf mysqld_exporter-0.14.0.linux-amd64.tar.gz -C /opt
cd /opt/
mv mysqld_exporter-0.14.0.linux-amd64/ mysqld_exporter
[root@easzlab-minio-01 ~]# vi /root/.my.cnf [root@easzlab-minio-01 ~]# cat /root/.my.cnf [client] user=mysql_exporter password=imnot007* [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 ~]# vi /etc/systemd/system/mysqld_exporter.service [root@easzlab-minio-01 ~]# cat /etc/systemd/system/mysqld_exporter.service [Unit] Description=Prometheus mysql Exporter After=network.target [Service] ExecStart=/opt/mysql_exporter/mysqld_exporter --config.my-cnf=/root/.my.cnf [Install] WantedBy=multi-user.target [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 ~]# systemctl enable --now mysqld_exporter.service Created symlink /etc/systemd/system/multi-user.target.wants/mysqld_exporter.service → /etc/systemd/system/mysqld_exporter.service. [root@easzlab-minio-01 ~]# systemctl status mysqld_exporter.service ● mysqld_exporter.service - Prometheus Node Exporter Loaded: loaded (/etc/systemd/system/mysqld_exporter.service; enabled; vendor preset: enabled) Active: active (running) since Tue 2022-09-27 03:46:25 UTC; 9s ago Main PID: 93879 (mysqld_exporter) Tasks: 6 (limit: 9442) Memory: 2.3M CGroup: /system.slice/mysqld_exporter.service └─93879 /opt/mysqld_exporter/mysqld_exporter --config.my-cnf=/root/.my.cnf Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.045Z caller=mysqld_exporter.go:277 level=info msg="Starting mysqld_exporter" version="(ver> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.046Z caller=mysqld_exporter.go:278 level=info msg="Build context" (gogo1.17.8,userroot@401> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.046Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=global_status Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.046Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=global_variabl> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.046Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=slave_status Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.046Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=info_schema.in> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.047Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=info_schema.in> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.047Z caller=mysqld_exporter.go:293 level=info msg="Scraper enabled" scraper=info_schema.qu> Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.047Z caller=mysqld_exporter.go:303 level=info msg="Listening on address" address=:9104 Sep 27 03:46:26 easzlab-minio-01 mysqld_exporter[93879]: ts=2022-09-27T03:46:26.053Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false [root@easzlab-minio-01 ~]#
访问本地mysql_exporter 9104端口可以看到相关监控数据
7.4、配置Prometheus
[root@monitoring prometheus]# vim prometheus.yml
[root@monitoring prometheus]# grep mysql -C 5 prometheus.yml #haproxy服务采集 - job_name: "haproxy-monitor-metrics" static_configs: - targets: ["172.16.88.151:9101","172.16.88.152:9101","172.16.88.153:9101"] #mysql服务采集 - job_name: "mysql-monitor-metrics" static_configs: - targets: ["172.16.88.170:9104"] #API Serevr节点发现 - job_name: 'kubernetes-apiservers-monitor' [root@monitoring prometheus]# ./promtool check config prometheus.yml Checking prometheus.yml SUCCESS: prometheus.yml is valid prometheus config file syntax [root@monitoring prometheus]# systemctl restart prometheus.service [root@monitoring prometheus]#
7.5、测试数据库
八、监控nginx与ingress-nginx-controller
8.1、下载nginx及相关的nginx工具
git clone https://github.com/vozlt/nginx-module-vts.git
wget http://nginx.org/download/nginx-1.20.2.tar.gz
tar xvf nginx-1.20.2.tar.gz
cd nginx-1.20.2/
mkdir /apps/nginx
8.2、安装nginx编译环境
apt update
apt -y install gcc make libpcre3 libpcre3-dev openssl libssl-dev zlib1g-dev
./configure --prefix=/apps/nginx \ --with-http_ssl_module \ --with-http_v2_module \ --with-http_realip_module \ --with-http_stub_status_module \ --with-http_gzip_static_module \ --with-pcre \ --with-file-aio \ --with-stream \ --with-stream_ssl_module \ --with-stream_realip_module \ --add-module=/root/nginx/nginx-module-vts/ make && make install
配置nginx，并把Prometheus页面代理到本地nginx，方便后面nginx监控采集监控指标
[root@easzlab-minio-01 ~]# grep -Ev "^$|^#|^[[:space:]]+#" /apps/nginx/conf/nginx.conf worker_processes 1; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; vhost_traffic_status_zone; server { listen 80; server_name localhost; location / { root html; index index.html index.htm; proxy_pass http://172.16.88.20:9090; } location /status { vhost_traffic_status_display; vhost_traffic_status_display_format html; } error_page 500 502 503 504 /50x.html; location = /50x.html { root html; } } } [root@easzlab-minio-01 ~]# /apps/nginx/sbin/nginx -t nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok nginx: configuration file /apps/nginx/conf/nginx.conf test is successful [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 ~]# /apps/nginx/sbin/nginx [root@easzlab-minio-01 ~]# ps -ef |grep nginx root 107010 1 0 06:59 ? 00:00:00 nginx: master process /apps/nginx/sbin/nginx nobody 107011 107010 0 06:59 ? 00:00:00 nginx: worker process root 107013 93785 0 06:59 pts/1 00:00:00 grep --color=auto nginx [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 ~]# netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 127.0.0.1:32905 0.0.0.0:* LISTEN 804/containerd tcp 0 0 127.0.0.1:3306 0.0.0.0:* LISTEN 93052/mysqld tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 107010/nginx: maste tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 703/systemd-resolve tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 911/sshd: /usr/sbin tcp 0 0 127.0.0.1:6010 0.0.0.0:* LISTEN 93695/sshd: root@pt tcp 0 0 127.0.0.1:6011 0.0.0.0:* LISTEN 101981/sshd: root@p tcp 0 0 127.0.0.1:33060 0.0.0.0:* LISTEN 93052/mysqld tcp6 0 0 :::9000 :::* LISTEN 1235/minio tcp6 0 0 :::9104 :::* LISTEN 93879/mysqld_export tcp6 0 0 :::22 :::* LISTEN 911/sshd: /usr/sbin tcp6 0 0 ::1:6010 :::* LISTEN 93695/sshd: root@pt tcp6 0 0 ::1:6011 :::* LISTEN 101981/sshd: root@p tcp6 0 0 :::9090 :::* LISTEN 1235/minio [root@easzlab-minio-01 ~]#
验证nginx反向代理
查看nginx访问统计页面
8.3、安装nginx-exporter
wget https://github.com/hnlq715/nginx-vts-exporter/releases/download/v0.10.3/nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
tar -xf nginx-vts-exporter-0.10.3.linux-amd64.tar.gz
mv nginx-vts-exporter-0.10.3.linux-amd64 nginx-vts-exporter
mv nginx-vts-exporter /opt/
[root@easzlab-minio-01 ~]# cat /etc/systemd/system/nginx-vts-exporter.service [Unit] Description=Prometheus nginx-vts Exporter After=network.target [Service] ExecStart=/opt/nginx-vts-exporter/nginx-vts-exporter --nginx.scrape_uri=http://172.16.88.170/status/format/json [Install] WantedBy=multi-user.target [root@easzlab-minio-01 ~]# [root@easzlab-minio-01 nginx-vts-exporter]# vi /etc/systemd/system/nginx-vts-exporter.service [root@easzlab-minio-01 nginx-vts-exporter]# systemctl start nginx-vts-exporter.service [root@easzlab-minio-01 nginx-vts-exporter]# systemctl status nginx-vts-exporter.service ● nginx-vts-exporter.service - Prometheus nginx-vts Exporter Loaded: loaded (/etc/systemd/system/nginx-vts-exporter.service; disabled; vendor preset: enabled) Active: active (running) since Tue 2022-09-27 07:35:50 UTC; 4s ago Main PID: 107137 (nginx-vts-expor) Tasks: 5 (limit: 9442) Memory: 1.3M CGroup: /system.slice/nginx-vts-exporter.service └─107137 /opt/nginx-vts-exporter/nginx-vts-exporter --nginx.scrape_uri=http://172.16.88.170/status/format/json Sep 27 07:35:50 easzlab-minio-01 systemd[1]: Started Prometheus nginx-vts Exporter. Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Starting nginx_vts_exporter (version=0.10.3, branch=HEAD, revision=8aa2881c7050d9b28f2312> Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Build context (go=go1.10, user=root@56ca8763ee48, date=20180328-05:47:47) Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Starting Server at : :9913 Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Metrics endpoint: /metrics Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Metrics namespace: nginx Sep 27 07:35:50 easzlab-minio-01 nginx-vts-exporter[107137]: 2022/09/27 07:35:50 Scraping information from : http://172.16.88.170/status/format/json [root@easzlab-minio-01 nginx-vts-exporter]# [root@easzlab-minio-01 nginx-vts-exporter]# [root@easzlab-minio-01 nginx-vts-exporter]# [root@easzlab-minio-01 nginx-vts-exporter]# netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 127.0.0.1:32905 0.0.0.0:* LISTEN 804/containerd tcp 0 0 127.0.0.1:3306 0.0.0.0:* LISTEN 93052/mysqld tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 107010/nginx: maste tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 703/systemd-resolve tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 911/sshd: /usr/sbin tcp 0 0 127.0.0.1:6010 0.0.0.0:* LISTEN 93695/sshd: root@pt tcp 0 0 127.0.0.1:6011 0.0.0.0:* LISTEN 101981/sshd: root@p tcp 0 0 127.0.0.1:33060 0.0.0.0:* LISTEN 93052/mysqld tcp6 0 0 :::9000 :::* LISTEN 1235/minio tcp6 0 0 :::9104 :::* LISTEN 93879/mysqld_export tcp6 0 0 :::22 :::* LISTEN 911/sshd: /usr/sbin tcp6 0 0 :::9913 :::* LISTEN 107137/nginx-vts-ex tcp6 0 0 ::1:6010 :::* LISTEN 93695/sshd: root@pt tcp6 0 0 ::1:6011 :::* LISTEN 101981/sshd: root@p tcp6 0 0 :::9090 :::* LISTEN 1235/minio [root@easzlab-minio-01 nginx-vts-exporter]#
8.4、配置Prometheus
[root@monitoring prometheus]# vim prometheus.yml [root@monitoring prometheus]# grep nginx-vts -C 5 prometheus.yml #mysql服务采集 - job_name: "mysql-monitor-metrics" static_configs: - targets: ["172.16.88.170:9104"] #nginx-vts服务采集 - job_name: "nginx-vts-monitor-metrics" static_configs: - targets: ["172.16.88.170:9913"] #API Serevr节点发现 - job_name: 'kubernetes-apiservers-monitor' [root@monitoring prometheus]# ./promtool check config prometheus.yml Checking prometheus.yml SUCCESS: prometheus.yml is valid prometheus config file syntax [root@monitoring prometheus]# systemctl restart prometheus.service [root@monitoring prometheus]#
8.5、导入nginx-vts模板
https://grafana.com/grafana/dashboards/2949
