docker-compose安装grafana+prometheus+loki 监控docker容器收集服务日志
使用docker-compose部署springboot项目后想找一个比较好用的日志收集工具，用elk感觉太重了，小项目承担不起，无意中发现loki，与grafana集成度很高，最重要的是使用成本低，记录下安装过程。
创建grafana，prometheus文件夹及配置文件
# 创建grafana文件夹
mkdir /docker/grafana/grafana
mkdir /docker/grafana/grafana/data
# 创建prometheus文件夹
mkdir /docker/grafana/prometheus
mkdir /docker/grafana/prometheus/data
mkdir /docker/grafana/prometheus/log
# 创建prometheus配置文件
vi /docker/grafana/prometheus/prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s
# 报警暂时不需要，后续可以添加
#alerting:
# alertmanagers:
# - static_configs:
# - targets:
# - alertmanager:9093
rule_files:
- "*rules.yml"
scrape_configs:
# prometheus
- job_name: 'prometheus'
static_configs:
- targets: ['host-ip:9090']
# 主机监控
- job_name: 'node'
static_configs:
- targets: ['host-ip:9100']
# docker资源监控
- job_name: 'cadvisor'
static_configs:
- targets: ['host-ip:8080']
# mysql
- job_name: 'mysql'
static_configs:
- targets: ['host-ip::9104']
labels:
instance: mysql
# springboot 项目监控
# springboot 监控需要项目中集成prometheus
# - job_name: 'test-server'
# metrics_path: '/actuator/prometheus'
# static_configs:
# - targets: ['tesete-server-ip:8081']
# - job_name: 'alertmanager'
# static_configs:
# - targets: ['alertmanager:9093']
创建loki 文件夹及配置文件
mkdir /docker/grafana/loki
mkdir /docker/grafana/loki/chunks
mkdir /docker/grafana/loki/rules
vi /docker/grafana/loki/local-config.yaml
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093
limits_config:
enforce_metric_name: false
reject_old_samples: true
reject_old_samples_max_age: 168h
per_stream_rate_limit: 100MB
per_stream_rate_limit_burst: 200MB
#ingestion_rate_mb: 32 #采集速率 32
#ingestion_burst_size_mb: 64 #单次采集大小 64
#max_entries_limit_per_query: 0 #查询返回的条目的最大行数，默认是5000，0表示无限制
#retention_period: 24h #全局保留期，超过这个时间的日志会被删除，具体删除规则在compactor里配置
#retention_stream: #局部保留期
#- selector: '{level="error"}'
# priority: 1
# period: 720h
table_manager:
retention_deletes_enabled: true
# 注意和索引存储时间一致，按1周为单位
retention_period: 1344h
创建compose文件
vi /docker/grafana/docker-compose.yml
version: "2"
networks:
prom:
services:
prometheus:
image: prom/prometheus:latest
container_name: prometheus
hostname: prometheus
restart: always
user: root
volumes:
# 挂载prometheus配置文件
- /docker/grafana/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
- /docker/grafana/prometheus/data:/prometheus
- /docker/grafana/prometheus/log:/var/log/
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--storage.tsdb.retention.time=7d'
ports:
- "9090:9090"
networks:
- prom
grafana:
image: grafana/grafana:latest
container_name: grafana
hostname: grafana
restart: always
user: root
volumes:
- /docker/grafana/grafana/data:/var/lib/grafana
ports:
- "3000:3000"
networks:
- prom
node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
hostname: exporter
restart: always
ports:
- "9100:9100"
networks:
- prom
cadvisor:
image: google/cadvisor:latest
container_name: cadvisor
hostname: cadvisor
restart: always
volumes:
# 挂载docker目录，为了监控docker使用情况，必须
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
ports:
- "8080:8080"
networks:
- prom
# 数据库监控
mysql-exporter:
image: prom/mysqld-exporter
container_name: mysql-exporter
restart: always
environment:
DATA_SOURCE_NAME: "root:root@(192.168.2.182:3306)/"
ports:
- "9104:9104"
volumes:
- /docker/service/logs:/Users/zhuzuopeng/Documents/dr-project/logs
loki:
image: grafana/loki:latest
container_name: loki
hostname: loki
restart: always
user: root
volumes:
# 挂载chunks文件夹及配置文件，loki收集到日志后会存储到chunks中
- /docker/grafana/loki/chunks:/loki/chunks
- /docker/grafana/loki/rules:/loki/rules
- /docker/grafana/loki/local-config.yaml:/etc/loki/local-config.yaml
ports:
- "3100:3100"
networks:
- prom
cd /docker/grafana
docker-compose up -d
收集日志
使用loki-docker-driver
docker 安装loki驱动收集日志
docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions
需要使用loki记录日志的docker-compose文件中替换日志收集驱动
logging:
driver: loki
options:
loki-url: "http://loki-server-ip:3100/api/prom/push"
loki-retries: "5"
loki-batch-size: "400"
使用promtail收集日志文件
开启promtail服务
version: "2"
services:
grafana_promtail:
container_name: promtail2
image: grafana/promtail:2.4.0
restart: always
user: root
volumes:
- /docker/test-service/logs:/var/log
- /docker/grafana/promatil/data:/var/log-position
- /docker/grafana/promatil/config.yml:/etc/promtail/config.yml
vi /docker/grafana/promatil/config.yml
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /var/log-position/positions.yaml # 记录日志上传的行号
# loki服务地址
client:
url: http://loki-server-ip:3100/loki/api/v1/push
scrape_configs:
- job_name: my-service
static_configs:
- targets:
- localhost
labels:
job: test-service
host: *.*.*.*
__path__: /var/log/*.log
好用的grafana模板下载
https://grafana.com/dashboards
Spring Boot Statistics
6756
1 Node Exporter for Prometheus Dashboard EN 20201010
11074
Docker and system monitoring
893
Docker Container & Host Metrics
10619
