K8S-ELK日志系统 之三：kafka集群

kafka： https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/3.0.0/kafka_2.12-3.0.0.tgz

1、创建kafka NFS

10.0.8.131:/data.kafka

2、用户和权限与一和二一致

3、新增nfs-provisioner

新增container和volume

 1 kind: Deployment
 2 apiVersion: apps/v1
 3 metadata:
 4   name: nfs-client-provisioner
 5   namespace: es
 6   labels:
 7     app: nfs-client-provisioner
 8 spec:
 9   replicas: 1
10   strategy:
11     type: Recreate
12   selector:
13     matchLabels:
14       app: nfs-client-provisioner
15   template:
16     metadata:
17       labels:
18         app: nfs-client-provisioner
19     spec:
20       serviceAccount: nfs-provisioner
21       containers:
22         - name: nfs-client-provisioner
23           #          image: quay.io/external_storage/nfs-client-provisioner:latest
24           image: easzlab/nfs-subdir-external-provisioner:v4.0.1
25           volumeMounts:
26             - name: nfs-client-root
27               mountPath: /persistentvolumes
28           env:
29             - name: PROVISIONER_NAME
30               value: nfs-provisioner
31             - name: NFS_SERVER
32               value: 10.0.8.131
33             - name: NFS_PATH
34               value: /data/elasticsearch
35         - name: nfs-zookeeper-provisioner
36           image: easzlab/nfs-subdir-external-provisioner:v4.0.1
37           volumeMounts:
38             - name: nfs-zookeeper-root
39               mountPath: /persistentvolumes
40           env:
41             - name: PROVISIONER_NAME
42               value: nfs-zookeeper-provisioner
43             - name: NFS_SERVER
44               value: 10.0.8.131
45             - name: NFS_PATH
46               value: /data/zookeeper
47         - name: nfs-kafka-provisioner
48           image: easzlab/nfs-subdir-external-provisioner:v4.0.1
49           volumeMounts:
50             - name: nfs-kafka-root
51               mountPath: /persistentvolumes
52           env:
53             - name: PROVISIONER_NAME
54               value: nfs-kafka-provisioner
55             - name: NFS_SERVER
56               value: 10.0.8.131
57             - name: NFS_PATH
58               value: /data/kafka
59       volumes:
60         - name: nfs-client-root
61           nfs:
62             server: 10.0.8.131
63             path: /data/elasticsearch
64         - name: nfs-zookeeper-root
65           nfs:
66             server: 10.0.8.131
67             path: /data/zookeeper
68         - name: nfs-kafka-root
69           nfs:
70             server: 10.0.8.131
71             path: /data/kafka
nfs-provisioner.yaml

3、新增storageclass

1 apiVersion: storage.k8s.io/v1
2 kind: StorageClass
3 metadata:
4   name: nfs-kafka-storage
5 provisioner: nfs-kafka-provisioner
6 parameters:
7   archiveOnDelete: "true"
storageclass.yaml

4、Dockerfile

 1 FROM openjdk:11
 2 ADD  kafka_2.12-3.0.0.tgz /
 3 COPY sources.list /etc/apt/
 4 COPY server.properties /kafka_2.12-3.0.0/config/
 5 COPY entrypoint.sh /
 6 RUN apt-get update -y &&  \
 7     apt-get install vim inetutils-ping net-tools telnet -y && \
 8     chmod 777 /tmp && \
 9     chmod +x entrypoint.sh
10 ENV PATH="/kafka_2.12-3.0.0/bin:${PATH}" \
11     KAFKA_VERSION=3.0.0
12     
13 ENTRYPOINT ["/entrypoint.sh"]
14 CMD ["kafka-server-start.sh","/kafka_2.12-3.0.0/config/server.properties"]
15 EXPOSE 9092
Dockerfile
 1 #!/bin/bash 
 2 
 3 CONFIGFILE_PATH="/kafka_2.12-3.0.0/config/server.properties"
 4 IFS=\,, read -a zkservers <<<"${ZKSERVERS}"
 5 zkconnect=''
 6 for i in ${zkservers[@]};do
 7   if [ "${zkconnect}x" == "x" ];then
 8     zkconnect="$i:$PORT"
 9     
10   else      
11     zkconnect="${zkconnect},$i:$PORT"
12   fi
13 done
14 echo "broker.id=`hostname|awk -F'-' '{print $2}'`">> ${CONFIGFILE_PATH}
15 echo "zookeeper.connect=${zkconnect}" >> ${CONFIGFILE_PATH}
16 echo "log.dirs=${LOG_DIR}" >> ${CONFIGFILE_PATH}
17 echo "listeners=PLAINTEXT://`hostname -i`:9092" >> ${CONFIGFILE_PATH}
18 
19 exec "$@"
entrypoint.sh
# 默认注释了源码镜像以提高 apt update 速度，如有需要可自行取消注释
deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free
# deb-src https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye main contrib non-free
deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-updates main contrib non-free
# deb-src https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-updates main contrib non-free

deb https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free
# deb-src https://mirrors.tuna.tsinghua.edu.cn/debian/ bullseye-backports main contrib non-free

deb https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free
# deb-src https://mirrors.tuna.tsinghua.edu.cn/debian-security bullseye-security main contrib non-free
sources.list
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
server.properties

传递到本地仓库：

docker build -t harbor.myland.com/baseimages/kafka-jdk11:3.0.0-20211119 .

docker push harbor.myland.com/baseimages/kafka-jdk11:3.0.0-20211119  

 5、statefulset.yaml

包含注册到zookeeper

 1 apiVersion: v1
 2 kind: Service
 3 metadata:
 4   name: kafka
 5   namespace: es
 6   labels:
 7     app: kafka
 8 spec:
 9   type: NodePort
10   ports:
11   - name: kafka
12     port: 9020
13     targetPort: 9020
14     nodePort: 30209
15   selector:
16     app: kafka
17 ---
18 apiVersion: apps/v1
19 kind: StatefulSet
20 metadata:
21   name: kafka
22   namespace: es
23 spec:
24   selector:
25     matchLabels:
26       app: kafka
27   serviceName: kafka
28   replicas: 3
29   template:
30     metadata:
31       labels:
32         app: kafka
33     spec:
34       containers:
35       - name: kafka
36         image: harbor.myland.com/baseimages/kafka-jdk11:3.0.0-20211119
37         imagePullPolicy: Always
38         env:
39         - name: LOG_DIR
40           value: "/data/kafka-logs"
41         - name: PORT
42           value: "2181"
43         - name: ZKSERVERS
44           value: "zookeeper-0.zk,zookeeper-1.zk,zookeeper-2.zk"
45 
46         ports:
47         - name: kafka
48           containerPort: 9020
49         volumeMounts:
50           - name: data
51             mountPath: /data
52         resources:
53           limits:
54             cpu: 200m
55             memory: 600Mi
56           requests:
57             cpu: 100m
58             memory: 100Mi
59         readinessProbe:
60           exec:
61             command: ["netstat", "-ant","grep","9200"]
62           initialDelaySeconds: 30
63           periodSeconds: 10
64           timeoutSeconds: 5
65       affinity:
66         podAntiAffinity:
67           requiredDuringSchedulingIgnoredDuringExecution:
68           - labelSelector:
69               matchExpressions:
70               - {key: app ,operator: In, values: ["kafka"]}
71             topologyKey: kubernetes.io/hostname   
72 
73   volumeClaimTemplates:
74   - metadata:
75       name: data
76       annotations:
77         volume.beta.kubernetes.io/storage-class: "nfs-kafka-storage"
78       namespace: es
79     spec:
80       accessModes: [ "ReadWriteOnce" ]
81       storageClassName: nfs-kafka-storage
82       resources:
83         requests:
84           storage: 5Gi
kafka_stateful.yaml

6、验证

 

 

posted @ 2021-12-19 15:15  西风发财  阅读(248)  评论(0)  编辑  收藏  举报