kubernetes ---- Loki收集K8s日志（文件类型日志）

由于公司K8s集群环境并不是很大的集群，所以日志收集使用的是Loki+Grafana的方案，使用helm的方式安装部署loki启动前修改配置文件已应用自定义配置来实现日志的收集，文件方式的日志收集等。

一、Loki的部署

部署helm；

https://www.yuque.com/kkkfree/ilecfh/tf6dai

helm拉取loki-stack；

# 加源
~] helm repo add grafana https://grafana.github.io/helm-charts
~] helm repo update
# 拉取2.0.0版本的loki，因为现在已有的grafana版本是5.0与最新版本的loki不兼容，所以使用老版本
~] helm fetch grafana/loki-stack --version v2.0.0 --untar --untardir .
~] cd loki-stack
# 生成 k8s 配置
loki-stack ] helm template loki . > loki.yaml

自定义部署文件

loki-stack ] mkdir conf
loki-stack ] cp loki.yaml{,.bak}
loki-stack ] sed -i 's/namespace: logs/namespace: default/g' loki.yaml
loki-stack ] vim loki.yaml
...
apiVersion: v1
kind: Service
metadata:
  name: loki
  namespace: logs
  labels:
    app: loki
    chart: loki-2.0.0
    release: loki
    heritage: Helm
  annotations:
    {}
spec:
# 端口类型修改为NodePort绑定到节点上的3100，我们这边使用的Nginx代理，所以不改也可以
  type: NodePort
  ports:
    - port: 3100
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
      nodePort: 3100
  selector:
    app: loki
    release: loki

# 搜索"storage"将emptyDir修改为pvc的名称，这里做日志持久化
      volumes:
        - name: config
          secret:
            secretName: loki
        - name: storage
          persistentVolumeClaim:
            claimName: logs-pvc

# 搜索"hostPath"找到定义的volumes在volumes下添加下列内容,文件方式记录日志的日志目录
# volumeMounts也要添加      
      - mountPath: /gensee/nginx
        name: nginx

      - hostPath:
          path: /gensee/nginx
          type: ""
        name: nginx


# 搜索kubernetes-pods-static,在这个ConfigMap里添加下列自定义的日志文件标签处理方式，在kubernetes-pods-static这一组的下列新增job_name
    - job_name: kubernetes-pods-name-nginx
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - source_labels:
        - __meta_kubernetes_pod_label_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /gensee/nginx/$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: __path__
    - job_name: kubernetes-pods-app-eschol-nginx
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        source_labels:
        - __meta_kubernetes_pod_label_name
      - source_labels:
        - __meta_kubernetes_pod_label_app
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /gensee/nginx/$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: __path__
    - job_name: kubernetes-pods-direct-controllers-nginx
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: drop
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /gensee/nginx/$1/*.log
        separator: /
        source_labels:
        - __meta_kuberenetes_pod_name
        target_label: __path__
    - job_name: kubernetes-pods-indirect-controller-nginx
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: .+
        separator: ''
        source_labels:
        - __meta_kubernetes_pod_label_name
        - __meta_kubernetes_pod_label_app
      - action: keep
        regex: '[0-9a-z-.]+-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
      - action: replace
        regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}'
        source_labels:
        - __meta_kubernetes_pod_controller_name
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /gensee/nginx/$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: __path__
    - job_name: kubernetes-pods-static-nginx
      pipeline_stages:
        - docker: {}
      kubernetes_sd_configs:
      - role: pod
      relabel_configs:
      - action: drop
        regex: ''
        source_labels:
        - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_label_component
        target_label: __service__
      - source_labels:
        - __meta_kubernetes_pod_node_name
        target_label: __host__
      - action: drop
        regex: ''
        source_labels:
        - __service__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - action: replace
        replacement: $1
        separator: /
        source_labels:
        - __meta_kubernetes_namespace
        - __service__
        target_label: job
      - action: replace
        source_labels:
        - __meta_kubernetes_namespace
        target_label: namespace
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: pod
      - action: replace
        source_labels:
        - __meta_kubernetes_pod_container_name
        target_label: container
      - replacement: /gensee/nginx/$1/*.log
        separator: /
        source_labels:
        - __meta_kubernetes_pod_name
        target_label: __path__
...
loki-stack ] mkdir pvc && cd pvc
pvc ] vim log-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: logs-pvc
  namespace: logs
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 200Gi
pvc ] kubectl apply -f log-pvc.yaml

自定义配置文件

# 获取loki的配置文件
~] kubectl get secret loki -o json -n logs | jq  -r '.data."loki.yaml"' | base64 -d > /root/loki-stack/conf/loki.yaml
~] vim /root/loki-stack/conf/loki.yaml
auth_enabled: false
chunk_store_config:
  max_look_back_period: 0s
compactor:
  shared_store: filesystem
  working_directory: /data/loki/boltdb-shipper-compactor
ingester:
  chunk_block_size: 262144
  chunk_idle_period: 3m
  chunk_retain_period: 1m
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
  max_transfer_retries: 0
limits_config:
  enforce_metric_name: false
  # 旧日志是否拒绝接收
  reject_old_samples: true
  # 多久以前的数据不接收
  reject_old_samples_max_age: 72h
  # 当你搭建完成promtail，并且启动发送日志到loki的时候很有可能会碰到这个错误，
  # 因为你要收集的日志太多了，超过了loki的限制，所以会报429，如果你要增加限制可以修改loki的配置文件,在limits_config中添加
  ingestion_rate_mb: 100
  # 每个用户允许摄入的突发大小(样本大小)。 单位MB。  
  # 突发大小指的是每个分发器本地速率限制器，即使在  
  # case的“全局”策略，并且应该至少设置为最大日志  
  # 单个推请求中期望的大小  
  ingestion_burst_size_mb: 200
schema_config:
  configs:
  - from: "2020-10-24"
    index:
      period: 24h
      prefix: index_
    object_store: filesystem
    schema: v11
    store: boltdb-shipper
server:
  http_listen_port: 3100
  grpc_server_max_recv_msg_size: 10388608
storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/boltdb-shipper-active
    cache_location: /data/loki/boltdb-shipper-cache
    cache_ttl: 24h
    shared_store: filesystem
  filesystem:
    directory: /data/loki/chunks
table_manager:
  retention_deletes_enabled: false
  retention_period: 0s

部署loki-stack

# 部署（如果要修改默认配置必须要修改一下yaml）
~] kubectl apply -f loki.yaml
# 使用自定义的配置文件
~] kubectl delete secret loki -n logs

自定义的Pod日志输出路径

...          
          volumeMounts:
          - name: nginx-log
            mountPath: /gensee/openresty/nginx/logs/
            subPathExpr: $(POD_NAME)
      volumes:
      - name: nginx-log
        hostPath:
          path: /gensee/nginx
          type: DirectoryOrCreate
...

反向代理Loki以让外部Grafana可连接Loki

~] vim nginx/conf/conf.d/loki.conf
server {
        listen       80;
        server_name  loki.general.com; # 替换成你的域名
        access_log  logs/loki.log;
        location / {
                proxy_pass http://loki.logs.svc.cluster.local:3100;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header Host $host;
                proxy_set_header X-Forward-For $proxy_add_x_forwarded_for;
                proxy_set_header Upgrade $http_upgrade;
                proxy_set_header Connection 'upgrade';
         }
}
~] kubectl delete cm nginx-confd -n nginx
~] kubectl create cm nginx-confd --from-file=nginx/conf/conf.d/loki.conf -n nginx