k8s安装efk日志收集fluent+elasticsearch+kibanav2
k8s安装EFK日志收集v2
基础安装nfs、rpcbind参考前一篇文章
es、kibana版本使用8.2.0,使用xpack.security.transport.ssl.verification_mode等加密
handlessSvc.yaml
[root@node81 logging]# cat handlessSvc.yaml
kind: Service
apiVersion: v1
metadata:
name: elasticsearch
namespace: logging
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
serviceAccount.yaml
[root@node81 logging]# cat serviceAccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-provisioner
namespace: logging
rbac.yaml
[root@node81 logging]# cat rbac.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-provisioner-runner
namespace: logging
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
- apiGroups: [""]
resources: ["services", "endpoints"]
verbs: ["get"]
- apiGroups: ["extensions"]
resources: ["podsecuritypolicies"]
resourceNames: ["nfs-provisioner"]
verbs: ["use"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-provisioner
namespace: logging
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: logging
roleRef:
kind: ClusterRole
name: nfs-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
namespace: logging
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-provisioner
namespace: logging
subjects:
- kind: ServiceAccount
name: nfs-provisioner
namespace: logging
roleRef:
kind: Role
name: leader-locking-nfs-provisioner
apiGroup: rbac.authorization.k8s.io
npv.yaml
[root@node81 logging]# cat npv.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
name: nfs-provisioner
namespace: logging
spec:
selector:
matchLabels:
app: nfs-provisioner
replicas: 1
strategy:
type: Recreate
template:
metadata:
labels:
app: nfs-provisioner
spec:
nodeSelector:
kubernetes.io/hostname: node81
serviceAccount: nfs-provisioner
containers:
- name: nfs-provisioner
image: registry.cn-hangzhou.aliyuncs.com/open-ali/nfs-client-provisioner:latest
imagePullPolicy: IfNotPresent
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: eslog/nfs #PROVISIONER_NAME是eslog/nfs，eslog/nfs需要跟后面的storageclass的provisinoer保持一致
- name: NFS_SERVER
value: 192.168.0.81 #这个需要写nfs服务端所在的ip地址，此处为k8s-elasticsearch地址
- name: NFS_PATH
value: /data/eslog #共享目录
volumes:
- name: nfs-client-root
nfs:
server: 192.168.0.81 #这个是nfs服务端的ip,大家需要写自己的nfs地址
path: /data/eslog
class.yaml
[root@node81 logging]# cat class.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: es-block-storage
namespace: logging
provisioner: eslog/nfs
es.yaml
[root@node81 logging]# cat es.yaml
# RBAC authn and authz
apiVersion: v1
kind: ServiceAccount
metadata:
name: elasticsearch
namespace: logging
labels:
app: elasticsearch
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: elasticsearch
namespace: logging
labels:
app: elasticsearch
rules:
- apiGroups:
- ""
resources:
- "services"
- "namespaces"
- "endpoints"
verbs:
- "get"
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: elasticsearch
namespace: logging
labels:
app: elasticsearch
subjects:
- kind: ServiceAccount
name: elasticsearch
namespace: logging
apiGroup: ""
roleRef:
kind: ClusterRole
name: elasticsearch
apiGroup: ""
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es-cluster
namespace: logging
labels:
app: elasticsearch
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
nodeSelector:
kube-soft: elasticsearch
serviceAccountName: elasticsearch
containers:
- name: elasticsearch
image: elasticsearch:8.2.0
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: 1000m #单个容器最多可使用1个CPU
requests:
cpu: 100m #单个容器最少保证有0.1个CPU
ports:
- containerPort: 9200
name: rest #与handless service一致
protocol: TCP
- containerPort: 9300
name: inter-node
protocol: TCP
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
env:
- name: cluster.name #集群名称
value: k8s-logs
- name: node.name #节点名称，通过matedata.name获取
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: discovery.seed_hosts #设置在Elasticsearch集群中节点相互连接的发现方法，由于都在同一个 namespace 下面，我们可以将其缩短为es-cluster-[0,1,2].elasticsearch
value: "es-cluster-0.elasticsearch,es-cluster-1.elasticsearch,es-cluster-2.elasticsearch"
- name: cluster.initial_master_nodes
value: "es-cluster-0,es-cluster-1,es-cluster-2"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m" #告诉JVM使用512MB的最小和最大堆
- name: network.host
value: "0.0.0.0" #设置所有服务器都能访问
- name: xpack.monitoring.collection.enabled
value: "true" #kibana获取es集群信息配置，不开启需安装metrics才能连接es和kibana
- name: xpack.security.transport.ssl.enabled
value: "true"
- name: xpack.security.transport.ssl.verification_mode
value: "certificate"
- name: xpack.security.transport.ssl.keystore.path
value: "/usr/share/elasticsearch/data/elastic-certificates.p12"
- name: xpack.security.transport.ssl.truststore.path
value: "/usr/share/elasticsearch/data/elastic-certificates.p12"
- name: xpack.security.enabled
value: "true"
initContainers: #这里定义了几个在主应用程序之前运行的Init 容器，这些初始容器按照定义的顺序依次执行，执行完成后才会启动主应用容器。
- name: fix-permissions
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
securityContext:
privileged: true
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
#第一个名为 fix-permissions 的容器用来运行 chown 命令，将 Elasticsearch 数据目录的用户和组更改为1000:1000（Elasticsearch 用户的 UID）。
#因为默认情况下，Kubernetes 用 root 用户挂载数据目录，这会使得 Elasticsearch 无法方法该数据目录
- name: increase-vm-max-map
image: busybox
imagePullPolicy: IfNotPresent
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
#第二个名为increase-vm-max-map 的容器用来增加操作系统对mmap计数的限制，默认情况下该值可能太低，导致内存不足的错误
- name: increase-fd-ulimit
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
#最后一个初始化容器是用来执行ulimit命令增加打开文件描述符的最大数量的。
#此外 Elastisearch Notes for Production Use 文档还提到了由于性能原因最好禁用 swap，当然对于 Kubernetes 集群而言，最好也是禁用 swap 分区的
volumeClaimTemplates:
- metadata:
name: data
labels:
app: elasticsearch
spec:
accessModes: [ "ReadWriteOnce" ] #只能被 mount到单个节点上进行读写
storageClassName: es-block-storage #需要提前创建该对象，我们这里使用的 NFS 作为存储后端，所以需要安装一个对应的 provisioner驱动
resources:
requests:
storage: 10Gi
kibana.yaml
[root@node81 logging]# cat kibana.yaml
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: logging
labels:
app: kibana
spec:
type: NodePort #为了测试方便，我们将 Service 设置为了 NodePort 类型
ports:
- port: 5601
selector:
app: kibana
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: logging
name: kibana-config
labels:
app: kibana
data:
kibana.yml: |-
server.host: 0.0.0.0
xpack.encryptedSavedObjects.encryptionKey: "lgFubCJ5pBdtbuTiR9iLlgFubCJ5pBdtbuTiR9iL"
elasticsearch:
hosts: ${ELASTICSEARCH_HOSTS}
username: ${ELASTICSEARCH_USER}
password: ${ELASTICSEARCH_PASSWORD}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: logging
labels:
app: kibana
spec:
replicas: 1
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
nodeSelector:
kube-soft: elasticsearch
containers:
- name: kibana
image: kibana:8.2.0 #kibana版本需要与es版本一致
imagePullPolicy: IfNotPresent
ports:
- containerPort: 5601
resources:
limits:
cpu: 1000m
requests:
cpu: 100m
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch:9200 #设置为handless service 的名称
- name: ELASTICSEARCH_USER
value: "kibana"
- name: ELASTICSEARCH_PASSWORD
value: "lgFubCJ5pBdtbuTiR0iL"
- name: SERVER_NAME
value: kibana
- name: "I18N_LOCALE"
value: "zh-CN"
volumeMounts:
- name: config
mountPath: /usr/share/kibana/config/kibana.yml
readOnly: true
subPath: kibana.yml
volumes:
- name: config
configMap:
name: kibana-config
fluentd.yaml
[root@node81 logging]# cat fluentd.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- ""
resources:
- pods
- namespaces
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
kind: ClusterRole
name: fluentd
apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
name: fluentd
namespace: logging
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: fluentd
namespace: logging
labels:
app: fluentd
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
app: fluentd
template:
metadata:
labels:
app: fluentd
spec:
serviceAccount: fluentd
serviceAccountName: fluentd
# tolerations: #设置容忍，允许部署到master节点，获取master数据
# - key: node-role.kubernetes.io/master
# effect: NoSchedule
nodeSelector:
kube-soft: saas
containers:
- name: fluentd
image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch #镜像选择不同，对应的配置文件方式也不同注意进入fluent容器调整配置
imagePullPolicy: IfNotPresent
env:
- name: FLUENT_ELASTICSEARCH_HOST
value: "elasticsearch"
- name: FLUENT_ELASTICSEARCH_PORT
value: "9200"
- name: FLUENT_ELASTICSEARCH_SCHEME
value: "http"
- name: FLUENT_ELASTICSEARCH_USER
value: "elastic"
- name: FLUENT_ELASTICSEARCH_PASSWORD
value: "dzCSRsplbYFf2p7jA4Iz"
resources:
limits:
memory: 512Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: varlog
mountPath: /var/log
- name: datadockerdatacontainers
mountPath: /data/dockerdata/containers
readOnly: true
terminationGracePeriodSeconds: 30
volumes:
- name: varlog
hostPath:
path: /var/log
- name: datadockerdatacontainers
hostPath:
path: /data/dockerdata/containers #宿主机docker文件目录,这里我有修改
安装es
[root@node81 logging]# kubectl create ns logging
[root@node81 logging]# kubectl apply -f handlessSvc.yaml
[root@node81 logging]# kubectl apply -f serviceAccount.yaml
[root@node81 logging]# kubectl apply -f rbac.yaml
[root@node81 logging]# kubectl apply -f npv.yaml
[root@node81 logging]# kubectl apply -f class.yaml
生成Xpack认证文件
[root@node81 logging]# docker run -it -d --name elastic-cret elasticsearch:8.2.0 /bin/bash
[root@node81 logging]# docker exec -it elastic-cret /bin/bash
elasticsearch@2b9709c1f2d9:~$ ./elasticsearch-certutil ca
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.
Please enter the desired output file [elastic-stack-ca.p12]:
Enter password for elastic-stack-ca.p12 :
-------------------------------------
elasticsearch@2b9709c1f2d9:~$ ./bin/elasticsearch-certutil cert --ca elastic-stack-ca.p12
This tool assists you in the generation of X.509 certificates and certificate
signing requests for use with SSL/TLS in the Elastic stack.
......
Enter password for CA (elastic-stack-ca.p12) :
Please enter the desired output file [elastic-certificates.p12]:
Enter password for elastic-certificates.p12 :
Certificates written to /usr/share/elasticsearch/elastic-certificates.p12
#以上都直接回车即可
[root@node81 logging]# docker cp elastic-cret:/usr/share/elasticsearch/elastic-certificates.p12 .
[root@node81 logging]# docker stop elastic-cret
[root@node81 logging]# docker rm elastic-cret
#将xpack文件复制到nfs目录下，然后在nfs上复制进容器目录
[root@node81 logging]# scp elastic-certificates.p12 root@192.168.0.81:/data/eslog/
安装es
[root@node81 logging]# kubectl apply -f es.yaml
创建es密码
[root@node81 logging]# kubectl exec es-cluster-0 -n logging -- bin/elasticsearch-setup-passwords auto -b
Changed password for user apm_system
PASSWORD apm_system = 5wg8JbmKOKiLMNty90l1
Changed password for user kibana_system
PASSWORD kibana_system = 1bT0U5RbPX1e9zGNlWFL
Changed password for user kibana
PASSWORD kibana = 1bT0U5RbPX1e9zGNlWFL
Changed password for user logstash_system
PASSWORD logstash_system = 1ihEyA5yAPahNf9GuRJ9
Changed password for user beats_system
PASSWORD beats_system = WEWDpPndnGvgKY7ad0T9
Changed password for user remote_monitoring_user
PASSWORD remote_monitoring_user = MOCszTmzLmEXQrPIOW4T
Changed password for user elastic
PASSWORD elastic = bbkrgVrsE3UAfs2708aO
安装kibana、fluentd
将es密码填入指定yaml位置
[root@node81 logging]# kubectl apply -f kibana.yaml
[root@node81 logging]# kubectl apply -f fluentd.yaml
参考：https://www.cnblogs.com/wubolive/p/15765671.html
https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/fluentd-daemonset-elasticsearch.yaml
