K8S-ELK日志系统之一：elasticseatch集群

生产环境中，我们应当把监控系统和日志系统部署在生产环境之外的自主环境或者Kubernetes集群之上，此处仅为练习。

k8s: v1.21.5

Docker :20.10.5

elasticsearch: 7.16.0（也可以选择自带jdk的版本）

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.16.0-no-jdk-amd64.deb

基础镜像：本地仓库：harbor.myland.com/baseimages/openjdk:11

1、创建NFS共享存储

mkdir /data/elasticsearch

echo "/data/elasticsearch *(rw,sync,no_root_squash,no_subtree_check)" >> /etc/exports

exportfs -arv

NFS地址：10.0.8.131:/data/elasticsearch

2、创建namespace

1 apiVersion: v1
2 kind: Namespace
3 metadata:
4    name: es
5    labels:
6      name: es

es_namespace.yaml

3、创建账号和角色，并绑定

 1 apiVersion: v1
 2 kind: ServiceAccount
 3 metadata:
 4   name: nfs-provisioner
 5   namespace: neo4j
 6 ---
 7 kind: ClusterRole
 8 apiVersion: rbac.authorization.k8s.io/v1
 9 metadata:
10    name: nfs-provisioner-runner
11 rules:
12    -  apiGroups: [""]
13       resources: ["persistentvolumes"]
14       verbs: ["get", "list", "watch", "create", "delete"]
15    -  apiGroups: [""]
16       resources: ["persistentvolumeclaims"]
17       verbs: ["get", "list", "watch", "update"]
18    -  apiGroups: ["storage.k8s.io"]
19       resources: ["storageclasses"]
20       verbs: ["get", "list", "watch"]
21    -  apiGroups: [""]
22       resources: ["events"]
23       verbs: ["list", "watch", "create", "update", "patch"]
24    -  apiGroups: [""]
25       resources: ["services", "endpoints"]
26       verbs: ["get","create","list", "watch","update"]
27    -  apiGroups: ["extensions"]
28       resources: ["podsecuritypolicies"]
29       resourceNames: ["nfs-provisioner"]
30       verbs: ["use"]
31 ---
32 kind: ClusterRoleBinding
33 apiVersion: rbac.authorization.k8s.io/v1
34 metadata:
35   name: run-nfs-provisioner
36 subjects:
37   - kind: ServiceAccount
38     name: nfs-provisioner
39     namespace: neo4j
40 roleRef:
41   kind: ClusterRole
42   name: nfs-provisioner-runner
43   apiGroup: rbac.authorization.k8s.io

sc_bind_role.yaml

4、创建nfs-provisioner

参考：https://blog.csdn.net/weixin_41004350/article/details/90168631

使用镜像：image: easzlab/nfs-subdir-external-provisioner:v4.0.1代替 quay.io/external_storage/nfs-client-provisioner:latest，该镜像会报selfLink empty，，导致无法创建pv

 1 kind: Deployment
 2 apiVersion: apps/v1
 3 metadata:
 4   name: nfs-client-provisioner
 5   namespace: es
 6   labels:
 7     app: nfs-client-provisioner
 8 spec:
 9   replicas: 1
10   strategy:
11     type: Recreate
12   selector:
13     matchLabels:
14       app: nfs-client-provisioner
15   template:
16     metadata:
17       labels:
18         app: nfs-client-provisioner
19     spec:
20       serviceAccount: nfs-provisioner
21       containers:
22         - name: nfs-client-provisioner
23           #image: quay.io/external_storage/nfs-client-provisioner:latest
24           image: easzlab/nfs-subdir-external-provisioner:v4.0.1
25           volumeMounts:
26             - name: nfs-client-root
27               mountPath: /persistentvolumes
28           env:
29             - name: PROVISIONER_NAME
30               value: nfs-provisioner
31             - name: NFS_SERVER
32               value: 10.0.8.131
33             - name: NFS_PATH
34               value: /data/elasticsearch
35       volumes:
36         - name: nfs-client-root
37           nfs:
38             server: 10.0.8.131
39             path: /data/elasticsearch

nfs-provissioner.yaml

3、创建storageclass

1 apiVersion: storage.k8s.io/v1
2 kind: StorageClass
3 metadata:
4   name: managed-nfs-storage
5 provisioner: nfs-provisioner
6 parameters:
7   archiveOnDelete: "true"

storageclass.yaml

绑定上一步创建的：nfs-provisioner

4、创建Dockerfile

 1 FROM harbor.myland.com/baseimages/openjdk:11
 2 COPY entrypoint.sh /
 3 COPY elasticsearch-7.16.0-no-jdk-amd64.deb /
 4 RUN cd / && dpkg -i elasticsearch-7.16.0-no-jdk-amd64.deb && \
 5     chown -R  elasticsearch /usr/share/elasticsearch  && \
 6     chown -R elasticsearch /etc/elasticsearch  && \
 7     chown -R elasticsearch /etc/default/elasticsearch && \
 8     chmod +x /entrypoint.sh
 9 ENTRYPOINT ["/entrypoint.sh"]
10 CMD ["/bin/bash","-c","while true;do sleep 5;done"]

Dockerfile

 1 #! /bin/bash
 2 chown -R elasticsearch /data/
 3 echo "cluster.name: ${CLUSTER_NAME}" > /etc/elasticsearch/elasticsearch.yml
 4 echo "node.name: node-`hostname |awk -F'-' '{print $2}'`" >> /etc/elasticsearch/elasticsearch.yml
 5 echo "network.host: `hostname -i`" >> /etc/elasticsearch/elasticsearch.yml
 6 echo "http.port: ${HTTP_PORT}" >> /etc/elasticsearch/elasticsearch.yml
 7 echo "discovery.seed_hosts: ${SEED_HOSTS}" >> /etc/elasticsearch/elasticsearch.yml
 8 echo "cluster.initial_master_nodes: ${MASTER_NODES} " >> /etc/elasticsearch/elasticsearch.yml
 9 echo "path.data: /data/lib/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
10 echo "path.logs: /data/log/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
11 /etc/init.d/elasticsearch start
12 
13 exec "$@"

entrypoint.sh

变量通过创建容器时传入

5、elasticsearch_statefulset.yaml

创建service，集群通过传入的变量，生成elasticsearch配置文件

  1 apiVersion: v1
  2 kind: Service
  3 metadata:
  4   name: es
  5   namespace: es
  6   labels:
  7     app: es
  8 spec:
  9   type: NodePort
 10 #  clusterIP: None
 11   ports:
 12   - name: es
 13     port: 9200
 14     targetPort: 9200
 15     nodePort: 30010
 16     protocol: TCP
 17   selector:
 18     app: es
 19 ---
 20 apiVersion: apps/v1
 21 kind: StatefulSet
 22 metadata:
 23   name: es
 24   namespace: es
 25 spec:
 26   selector:
 27     matchLabels:
 28       app: es
 29   serviceName: es
 30   replicas: 3
 31   template:
 32     metadata:
 33       labels:
 34         app: es
 35         ctrl: pod-antiaffinity-required
 36     spec:
 37       initContainers:
 38         - name: sysctl
 39           image: busybox
 40           imagePullPolicy: Always
 41           securityContext:
 42             privileged: true
 43           command: ['sh', '-c', 'sysctl -w vm.max_map_count=262144']
 44       containers:
 45       - name: es
 46         image: harbor.myland.com/baseimages/es:v7.16.0
 47         imagePullPolicy: Always
 48         env:
 49         - name: CLUSTER_NAME
 50           value: "es"
 51         - name: HTTP_PORT
 52           value: "9200"
 53         - name: RECOVER_NODES
 54           value: "2"
 55         - name: ACTION_DESTRUCTIVE
 56           value: "TRUE"
 57         - name: SEED_HOSTS
 58           value: "['es-0.es','es-1.es','es-2.es']"
 59         - name: MASTER_NODES
 60           value: "['node-0','node-1','node-2']"
 61 
 62         ports:
 63         - name: es
 64           containerPort: 9200
 65         volumeMounts:
 66           - name: data
 67             mountPath: /data
 68         resources:
 69           limits:
 70             cpu: 400m
 71             memory: 1500Mi
 72           requests:
 73             cpu: 300m
 74             memory: 1000Mi
 75         livenessProbe:
 76           exec:
 77             command: ["netstat", "-ant","grep","9200"]
 78           initialDelaySeconds: 60
 79           periodSeconds: 10
 80           timeoutSeconds: 5
 81         readinessProbe:
 82           httpGet:
 83             port: 9200
 84             path: /_cat
 85           initialDelaySeconds: 1
 86           periodSeconds: 3
 87       affinity:
 88         podAntiAffinity:
 89           requiredDuringSchedulingIgnoredDuringExecution:
 90           - labelSelector:
 91               matchExpressions:
 92               - {key: app ,operator: In, values: ["es"]}
 93 #              - key: ctrl
 94 #                operator: In
 95 #                values: ["pod-antiaffinity-required"]
 96             topologyKey: kubernetes.io/hostname   
 97 
 98 
 99   volumeClaimTemplates:
100   - metadata:
101       name: data
102       annotations:
103         volume.beta.kubernetes.io/storage-class: "managed-nfs-storage"
104       namespace: es
105     spec:
106       accessModes: [ "ReadWriteOnce" ]
107       storageClassName: managed-nfs-storage
108       resources:
109         requests:
110           storage: 5Gi