【K8s日志】使用 Fluentd 收集 K8s 集群日志说明
fluentd部署清单示例
参考：https://github.com/kubernetes/kubernetes/tree/master/cluster/addons/fluentd-elasticsearch
kind: Namespace
apiVersion: v1
metadata:
name: logging
labels:
k8s-app: logging
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
# 创建优先级类
apiVersion: scheduling.k8s.io/v1beta1
kind: PriorityClass
metadata:
name: fluentd-priority
namespace: logging
value: 1000000
globalDefault: false
description: ""
---
# 创建服务账号
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd-es
namespace: logging
labels:
k8s-app: fluentd-es
addonmanager.kubernetes.io/mode: Reconcile
---
# 创建集群角色
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: fluentd-es
labels:
k8s-app: fluentd-es
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- ""
resources:
- "namespaces"
- "pods"
verbs:
- "get"
- "watch"
- "list"
---
# 创建集群角色绑定
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: fluentd-es
labels:
k8s-app: fluentd-es
addonmanager.kubernetes.io/mode: Reconcile
subjects:
- kind: ServiceAccount
name: fluentd-es
namespace: logging
apiGroup: ""
roleRef:
kind: ClusterRole
name: fluentd-es
apiGroup: ""
---
# 创建fluentd配置
kind: ConfigMap
apiVersion: v1
metadata:
name: fluentd-es-config
namespace: logging
labels:
addonmanager.kubernetes.io/mode: Reconcile
data:
#------系统配置参数-----
system.conf: |-
<system>
root_dir /tmp/fluentd-buffers/
</system>
#------Kubernetes 容器日志收集配置------
containers.input.conf: |-
<source>
@id fluentd-containers.log
@type tail
path /var/log/containers/*.log
pos_file /var/log/es-containers.log.pos
tag raw.kubernetes.*
read_from_head true
<parse>
@type multi_format
<pattern>
format json
time_key time
time_format %Y-%m-%dT%H:%M:%S.%NZ
</pattern>
<pattern>
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
time_format %Y-%m-%dT%H:%M:%S.%N%:z
</pattern>
</parse>
</source>
<match raw.kubernetes.**>
@id raw.kubernetes
@type detect_exceptions
remove_tag_prefix raw
message log
stream stream
multiline_flush_interval 5
max_bytes 500000
max_lines 1000
</match>
<filter **>
@id filter_concat
@type concat
key message
multiline_end_regexp /\n$/
separator ""
</filter>
<filter kubernetes.**>
@id filter_kubernetes_metadata
@type kubernetes_metadata
</filter>
<filter kubernetes.**>
@id filter_parser
@type parser
key_name log
reserve_data true
remove_key_name_field true
<parse>
@type multi_format
<pattern>
format json
</pattern>
<pattern>
format none
</pattern>
</parse>
</filter>
#------系统日志收集-------
system.input.conf: |-
<source>
@id journald-docker
@type systemd
matches [{ "_SYSTEMD_UNIT": "docker.service" }]
<storage>
@type local
persistent true
path /var/log/journald-docker.pos
</storage>
read_from_head true
tag docker
</source>
<source>
@id journald-container-runtime
@type systemd
matches [{ "_SYSTEMD_UNIT": "{{ fluentd_container_runtime_service }}.service" }]
<storage>
@type local
persistent true
path /var/log/journald-container-runtime.pos
</storage>
read_from_head true
tag container-runtime
</source>
<source>
@id journald-kubelet
@type systemd
matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
<storage>
@type local
persistent true
path /var/log/journald-kubelet.pos
</storage>
read_from_head true
tag kubelet
</source>
<source>
@id journald-node-problem-detector
@type systemd
matches [{ "_SYSTEMD_UNIT": "node-problem-detector.service" }]
<storage>
@type local
persistent true
path /var/log/journald-node-problem-detector.pos
</storage>
read_from_head true
tag node-problem-detector
</source>
<source>
@id kernel
@type systemd
matches [{ "_TRANSPORT": "kernel" }]
<storage>
@type local
persistent true
path /var/log/kernel.pos
</storage>
<entry>
fields_strip_underscores true
fields_lowercase true
</entry>
read_from_head true
tag kernel
</source>
#------输出到 ElasticSearch 配置------
output.conf: |-
<match **>
@id elasticsearch
@type elasticsearch
@log_level info
type_name _doc
include_tag_key true
host elasticsearch-logging #改成自己的 ElasticSearch 地址
port 9200
logstash_format true
logstash_prefix k8s
logst
<buffer>
@type file
path /var/log/fluentd-buffers/kubernetes.system.buffer
flush_mode interval
retry_type exponential_backoff
flush_thread_count 5
flush_interval 8s
retry_forever
retry_max_interval 30
chunk_limit_size 5M
queue_limit_length 10
overflow_action block
compress gzip #开启gzip提高日志采集性能
</buffer>
</match>
---
# 创建外部elasticsearch集群服务
apiVersion: v1
kind: Service
metadata:
name: elasticsearch-logging
namespace: logging
spec:
ports:
- port: 9200
protocol: TCP
targetPort: 9200
---
kind: Endpoints
apiVersion: v1
metadata:
name: elasticsearch-logging
namespace: logging
subsets:
- addresses:
- ip: 10.10.101.1
- ip: 10.10.101.2
- ip: 10.10.101.3
ports:
- port: 9200
---
# 创建节点日志代理
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd-es
namespace: logging
labels:
k8s-app: fluentd-es
version: v2.7.0
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: fluentd-es
version: v2.7.0
template:
metadata:
labels:
k8s-app: fluentd-es
version: v2.7.0
#此注释确保如果节点被驱逐，fluentd不会被驱逐
#支持关键的基于pod注释的优先级方案。
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: fluentd-priority #给 Fluentd 设置优先级资源
serviceAccountName: fluentd-es #给 Fluentd 分配权限账户
#设置容忍所有污点，这样可以收集所有节点日志如 Master 节点一般都被设污，不设置无法在其节点启动 fluentd。
tolerations:
- operator: "Exists"
containers:
- name: fluentd-es
image: 10.10.11.1/logging/fluentd:v2.7.0
env:
- name: FLUENTD_ARGS
value: --no-supervisor -q #不启用管理，-q 命令用平静时期于减少warn级别日志（-qq：减少error日志）
resources:
limits:
cpu: 100m
memory: 200Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: config-volume
mountPath: /etc/fluent/config.d
terminationGracePeriodSeconds: 30 #Kubernetes 将会给应用发送SIGTERM信号,用来优雅地关闭应用
volumes:
- name: varlog #将 Kubernetes 节点服务器日志目录挂入
hostPath:
path: /var/log
- name: varlibdockercontainers #挂入 Docker 容器日志目录
hostPath:
path: /var/lib/docker/containers
- name: config-volume #挂入 Fluentd 的配置参数
configMap:
name: fluentd-es-config
nodeSelector:
logging: "yes"
elasticsearch部署清单（生产环境需考虑安全和存储持久化）
apiVersion: v1
kind: Service
metadata:
name: elasticsearch-logging
namespace: logging
labels:
k8s-app: elasticsearch-logging
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "Elasticsearch"
spec:
clusterIP: None
ports:
- name: db
port: 9200
protocol: TCP
targetPort: 9200
- name: transport
port: 9300
protocol: TCP
targetPort: 9300
publishNotReadyAddresses: true
selector:
k8s-app: elasticsearch-logging
sessionAffinity: None
type: ClusterIP
---
# RBAC authn and authz
apiVersion: v1
kind: ServiceAccount
metadata:
name: elasticsearch-logging
namespace: logging
labels:
k8s-app: elasticsearch-logging
addonmanager.kubernetes.io/mode: Reconcile
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: elasticsearch-logging
labels:
k8s-app: elasticsearch-logging
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- ""
resources:
- "services"
- "namespaces"
- "endpoints"
verbs:
- "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: elasticsearch-logging
labels:
k8s-app: elasticsearch-logging
addonmanager.kubernetes.io/mode: Reconcile
subjects:
- kind: ServiceAccount
name: elasticsearch-logging
namespace: logging
apiGroup: ""
roleRef:
kind: ClusterRole
name: elasticsearch-logging
apiGroup: ""
---
# Elasticsearch deployment itself
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: elasticsearch-logging
namespace: logging
labels:
k8s-app: elasticsearch-logging
version: v7.10.2
addonmanager.kubernetes.io/mode: Reconcile
spec:
serviceName: elasticsearch-logging
replicas: 2
selector:
matchLabels:
k8s-app: elasticsearch-logging
version: v7.10.2
template:
metadata:
labels:
k8s-app: elasticsearch-logging
version: v7.10.2
spec:
serviceAccountName: elasticsearch-logging
containers:
- image: quay.io/fluentd_elasticsearch/elasticsearch:v7.10.2
name: elasticsearch-logging
imagePullPolicy: Always
resources:
# need more cpu upon initialization, therefore burstable class
limits:
cpu: 1000m
memory: 3Gi
requests:
cpu: 100m
memory: 3Gi
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
livenessProbe:
tcpSocket:
port: transport
initialDelaySeconds: 5
timeoutSeconds: 10
readinessProbe:
tcpSocket:
port: transport
initialDelaySeconds: 5
timeoutSeconds: 10
volumeMounts:
- name: elasticsearch-logging
mountPath: /data
env:
- name: "NAMESPACE"
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: "MINIMUM_MASTER_NODES"
value: "1"
volumes:
- name: elasticsearch-logging
emptyDir: {}
# Elasticsearch requires vm.max_map_count to be at least 262144.
# If your OS already sets up this number to a higher value, feel free
# to remove this init container.
initContainers:
- image: alpine:3.6
command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
name: elasticsearch-logging-init
securityContext:
privileged: true
kibana部署清单
apiVersion: v1
kind: Service
metadata:
name: kibana-logging
namespace: logging
labels:
k8s-app: kibana-logging
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/name: "Kibana"
spec:
ports:
- port: 5601
protocol: TCP
targetPort: ui
selector:
k8s-app: kibana-logging
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana-logging
namespace: logging
labels:
k8s-app: kibana-logging
addonmanager.kubernetes.io/mode: Reconcile
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kibana-logging
template:
metadata:
labels:
k8s-app: kibana-logging
spec:
securityContext:
seccompProfile:
type: RuntimeDefault
containers:
- name: kibana-logging
image: docker.elastic.co/kibana/kibana-oss:7.10.2
resources:
# need more cpu upon initialization, therefore burstable class
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch-logging:9200
- name: SERVER_NAME
value: kibana-logging
- name: SERVER_BASEPATH
value: /api/v1/namespaces/logging/services/kibana-logging/proxy
- name: SERVER_REWRITEBASEPATH
value: "false"
ports:
- containerPort: 5601
name: ui
protocol: TCP
livenessProbe:
httpGet:
path: /api/status
port: ui
initialDelaySeconds: 5
timeoutSeconds: 10
readinessProbe:
httpGet:
path: /api/status
port: ui
initialDelaySeconds: 5
timeoutSeconds: 10
