日志集中管理方案（grafana + loki + promtail + rsyslog）

参考资料

https://bandandevopsjourney.hashnode.dev/devopsday-74-creating-a-dashboard-using-grafana-with-integration-of-loki-and-promtail
https://zhuanlan.zhihu.com/p/602336166
https://www.jianshu.com/p/e82e4c93d839
https://blog.frognew.com/2023/05/loki-05-promtail-exmaples.html
https://docs.youdianzhishi.com/k8s/logging/loki/overview/
https://alexandre.deverteuil.net/post/syslog-relay-for-loki/
https://cloud.tencent.com/developer/article/2378441

1. syslog日志输出接口

# 输出级别 level, 用于控制输出哪些级别的日志
0 emerg: Emergency（紧急）
1 alert: Alerts（报警）
2 crit: Critical (关键）
3 err: Errors（错误）
4 warn: Warnings（警告）
5 notice: Notification（通知）
6 info: Information（消息）
7 debug: Debugging（调试）

# 设备层级 facility, 用于标识谁输出的
auth: 身份验证相关的消息（登录时）
cron: 进程或应用调度相关的消息
daemon: 守护进程相关的消息（内部服务器）
kernel: 内核相关的消息
mail: 内部邮件服务器相关的消息
syslog: syslog 守护进程本身相关的消息
lpr: 打印服务相关的消息
local0 – local7: 用户自定义的消息（local7通常被Cisco和Windows服务器使用）

2. docker-compose官方示例

# https://raw.githubusercontent.com/grafana/loki/v2.9.1/production/docker-compose.yaml

version: "3"

networks:
  loki:

services:
  loki:
    image: grafana/loki:2.9.0
    ports:
      - "3100:3100"
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - loki

  promtail:
    image: grafana/promtail:2.9.0
    volumes:
      - /var/log:/var/log
    command: -config.file=/etc/promtail/config.yml
    networks:
      - loki

  grafana:
    environment:
      - GF_PATHS_PROVISIONING=/etc/grafana/provisioning
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
    entrypoint:
      - sh
      - -euc
      - |
        mkdir -p /etc/grafana/provisioning/datasources
        cat <<EOF > /etc/grafana/provisioning/datasources/ds.yaml
        apiVersion: 1
        datasources:
        - name: Loki
          type: loki
          access: proxy
          orgId: 1
          url: http://loki:3100
          basicAuth: false
          isDefault: true
          version: 1
          editable: false
        EOF
        /run.sh
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    networks:
      - loki

3. 自定义compose

# cat > compose-grafana.yml <<EOF
version: '3'
  
services:
  grafana:
    image: grafana/grafana:10.2.3-ubuntu
    container_name: grafana
    restart: unless-stopped
    volumes:
      - grafana:/usr/share/grafana
      - grafana-data:/var/lib/grafana
    networks:
      - grafana-net
    ports:
     - 29300:3000

  loki:
    image: grafana/loki:2.9.1
    container_name: loki
    restart: unless-stopped
    volumes:
      - loki:/etc/loki
    networks:
      - grafana-net
    ports:
      - 3100:3100

  promtail:
    image: grafana/promtail:2.9.0
    restart: unless-stopped
    container_name: promtail
    volumes:
      - promtail:/etc/promtail
      - /var/log/rsyslog:/var/log/rsyslog
    networks:
      - grafana-net
    ports:
      - 9080:9080
      #- 1514:1514
      #- 1514:1514/udp

volumes:
  grafana:
    name: grafana
    #external: true
  grafana-data:
    name: grafana-data
    #external: true
  loki:
    name: loki
    #external: true
  promtail:
    name: promtail
    #external: true

networks:
  grafana-net:
    driver: bridge
    name: grafana-net
    ipam:
      driver: default
      config:
        - subnet: 172.18.14.0/24
          gateway: 172.18.14.1
EOF

4. promtail处理rsyslog日志

# Available Labels
__syslog_connection_ip_address: The remote IP address.
__syslog_connection_hostname: The remote hostname.
__syslog_message_severity: The syslog severity parsed from the message. Symbolic name as per syslog_message.go.
__syslog_message_facility: The syslog facility parsed from the message. Symbolic name as per syslog_message.go and syslog(3).
__syslog_message_hostname: The hostname parsed from the message.
__syslog_message_app_name: The app-name field parsed from the message.
__syslog_message_proc_id: The procid field parsed from the message.
__syslog_message_msg_id: The msgid field parsed from the message.
__syslog_message_sd_<sd_id>[_<iana_enterprise_id>]_<sd_name>: The structured-data field parsed from the message. 
The data field [custom@99770 example="1"] becomes __syslog_message_sd_custom_99770_example.

cat > /var/lib/docker/volumes/promtail/_data/config.yml <<EOF
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: network_log
  static_configs:
  - targets:
      - localhost
    labels:
      job: network_log
      __path__: /var/log/rsyslog/*log*
  
  relabel_configs:
    - source_labels: ['__syslog_message_severity']
      target_label: 'level'
    - source_labels: ['__syslog_message_facility']
      target_label: 'facility'
    - source_labels: ['__syslog_connection_ip_address']
      target_label: 'ip'
    - source_labels: ['__syslog_connection_hostname']
      target_label: 'host'
EOF

4. loki增加数据保留时间

# cat > /var/lib/docker/volumes/loki/_data/local-config.yaml <<EOF
auth_enabled: false

server:
  http_listen_port: 3100

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory: /loki/chunks
      rules_directory: /loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

limits_config:
  reject_old_samples: true
  reject_old_samples_max_age: 2160h

table_manager:
  retention_deletes_enabled: true
  retention_period: 2160h
EOF