用k3s搭建kafka集群

参考:https://blog.csdn.net/weixin_45669656/article/details/108667279?spm=1001.2101.3001.6650.1&utm_medium=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-1.no_search_link&depth_1-utm_source=distribute.pc_relevant.none-task-blog-2%7Edefault%7ECTRLIST%7Edefault-1.no_search_link

创建我的命名空间 zzh-k3s-namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  name: zzh-k3s
  labels:
    name: zzh-k3s

部署zookeeper 文件名zookeeper-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: zk-headless
  namespace: zzh-k3s
  labels:
    app: zk
spec:
  ports:
  - port: 2888
    name: server
  - port: 3888
    name: leader-election
  clusterIP: None   #指定无头服务，需要对外暴露自行创建一个service
  selector:
    app: zk
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: zk-pdb
  namespace: zzh-k3s
spec:
  selector:
    matchLabels:
      app: zk
  minAvailable: 2
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zk     #指定的zookeeper名称会在kafka.yaml里面用来配置和kafka连接，会创建pod为zk-0,zk-1,zk-2......这里只创建了三个
  namespace: zzh-k3s
spec:
  selector:
    matchLabels:
      app: zk
  serviceName: zk-headless
  replicas: 3
  updateStrategy:
    type: RollingUpdate # K8s 会将 StatefulSet 管理的 pod 分批次逐步替换掉
  podManagementPolicy: OrderedReady # 设置为Parallel这样Pod的创建就不必等待，而是会同时创建、同时删除
  template:
    metadata:
      labels:
        app: zk
    spec:
      containers:
      - name: k8s-zk
        image: mirrorgooglecontainers/kubernetes-zookeeper:1.0-3.4.10   #需要指定自己的镜像，可去阿里云上下载zookeeper镜像
        imagePullPolicy: IfNotPresent
        resources:
          requests:
            memory: "256Mi"
            cpu: "100m"
          limits:
            memory: "256Mi"
            cpu: "100m"
        ports:
        - containerPort: 2181
          name: client
        - containerPort: 2888
          name: server
        - containerPort: 3888
          name: leader-election
        command:
        - sh
        - -c
        - "start-zookeeper \
          --servers=5 \
          --data_dir=/var/lib/zookeeper/data \
          --data_log_dir=/var/lib/zookeeper/data/log \
          --conf_dir=/opt/zookeeper/conf \
          --client_port=2181 \
          --election_port=3888 \
          --server_port=2888 \
          --tick_time=2000 \
          --init_limit=10 \
          --sync_limit=5 \
          --heap=512M \
          --max_client_cnxns=60 \
          --snap_retain_count=3 \
          --purge_interval=12 \
          --max_session_timeout=40000 \
          --min_session_timeout=4000 \
          --log_level=INFO"
        - "zkGenConfig.sh && exec zkServer.sh start-foreground"
        readinessProbe:
          exec:
            command:
            - sh
            - -c
            - "zookeeper-ready 2181"
          initialDelaySeconds: 10
          timeoutSeconds: 5
        livenessProbe:
          exec:
            command:
            - sh
            - -c
            - "zookeeper-ready 2181"
          initialDelaySeconds: 10
          timeoutSeconds: 5
      securityContext:
        runAsUser: 1000
        fsGroup: 1000

测试1:

测试2:

测试3:

部署kafka kafka-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: kafka-svc
  namespace: zzh-k3s
  labels:
    app: kafka
spec:
  ports:
  - port: 9093
    targetPort: 9093
    nodePort: 30093
  selector:
    app: kafka
  type: NodePort
---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: kafka-pdb
  namespace: zzh-k3s
spec:
  selector:
    matchLabels:
      app: kafka
  minAvailable: 2
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka
  namespace: zzh-k3s
spec:
  selector:
    matchLabels:
      app: kafka
  serviceName: kafka-svc
  replicas: 3
  template:
    metadata:
      labels:
        app: kafka
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            - labelSelector:
                matchExpressions:
                  - key: "app"
                    operator: In
                    values: 
                    - kafka
              topologyKey: "kubernetes.io/hostname"
        podAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 1
               podAffinityTerm:
                 labelSelector:
                    matchExpressions:
                      - key: "app"
                        operator: In
                        values: 
                        - zk
                 topologyKey: "kubernetes.io/hostname"
      terminationGracePeriodSeconds: 300
      securityContext:
        runAsUser: 1000
        fsGroup: 1000
      containers:
      - name: k8skafka
        imagePullPolicy: IfNotPresent
        image: registry.cn-hangzhou.aliyuncs.com/ccgg/k8skafka:v1  #可自行去阿里云拉去对应镜像
        ports:
        - containerPort: 9093
          name: server
        command:
        - sh
        - -c
        - "exec kafka-server-start.sh /opt/kafka/config/server.properties --override broker.id=${HOSTNAME##*-} \   
          --override listeners=PLAINTEXT://:9093 \
          --override zookeeper.connect=zk-0.zk-headless.zzh-k3s.svc.cluster.local:2181,zk-1.zk-headless.zzh-k3s.svc.cluster.local:2181,zk-2.zk-headless.zzh-k3s.svc.cluster.local:2181 \   #这里配置了与zookeeper进行连接，非常重要。格式为：pod名.zookeeper的service名.名称空间.svc.cluster.local:2181
          --override log.dir=/var/lib/kafka "
        readinessProbe:
          exec:
           command: 
            - sh 
            - -c 
            - "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server=localhost:9093"
          initialDelaySeconds: 10    # 容器启动后多久开始探测
          timeoutSeconds: 2          # 表示容器必须在2s内做出相应反馈给probe，否则视为探测失败
          periodSeconds: 30          # 探测周期，每30s探测一次
          successThreshold: 1        # 连续探测1次成功表示成功
          failureThreshold: 3        # 连续探测3次失败表示失败

部署成功后查看pod情况：

验证kafka是否可用：

1、进入kafka-0命令: kubectl exec -it kafka-0 -nzzh-k3s bash
进入容器目录：cd /opt/kafka/config

2、创建一个名为aaa的topc命令：kafka-topics.sh --create --topic aaa --zookeeper zk-0.zk-headless.zzh-k3s.svc.cluster.local:2181,zk-1.zk-headless.zzh-k3s.svc.cluster.local:2181,zk-2.zk-headless.zzh-k3s.svc.cluster.local:2181 --partitions 3 --replication-factor 2

结果为：
Created topic “aaa”.

3、进入topic为aaa的生产者消息中心：kafka-console-consumer.sh --topic aaa --bootstrap-server localhost:9093