k8s部署zookeeper集群

zookeeper-ns.yaml

apiVersion: v1
kind: Namespace
metadata: 
  name: zookeeper

zookeeper-persistentvolume.yaml

apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-1
  namespace: zookeeper
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce 
  nfs:
    server: 192.168.174.102
    path: /data/k8s-data//zookeeper/zookeeper-datadir-1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-2
  namespace: zookeeper
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.174.102
    path: /data/k8s-data//zookeeper/zookeeper-datadir-1 

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: zookeeper-datadir-pv-3
  namespace: zookeeper
spec:
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  nfs:
    server: 192.168.174.102
    path: /data/k8s-data//zookeeper/zookeeper-datadir-1 

zookeeper-persistentvolumeclaim.yaml

---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-1
  namespace: zookeeper
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-1
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-2
  namespace: zookeeper
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-2
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zookeeper-datadir-pvc-3
  namespace: zookeeper
spec:
  accessModes:
    - ReadWriteOnce
  volumeName: zookeeper-datadir-pv-3
  resources:
    requests:
      storage: 5Gi

zookeeper.yaml

apiVersion: v1
kind: Service
metadata:
  name: zookeeper1
  namespace: zookeeper
spec:
  type: NodePort        
  ports:
    - name: client
      port: 2181
      nodePort: 42181
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper2
  namespace: zookeeper
spec:
  type: NodePort        
  ports:
    - name: client
      port: 2181
      nodePort: 42182
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "2"
---
apiVersion: v1
kind: Service
metadata:
  name: zookeeper3
  namespace: zookeeper
spec:
  type: NodePort        
  ports:
    - name: client
      port: 2181
      nodePort: 42183
    - name: followers
      port: 2888
    - name: election
      port: 3888
  selector:
    app: zookeeper
    server-id: "3"
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: zookeeper1
  namespace: zookeeper
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "1"
    spec:
      containers:
        - name: server
          image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "1"
            - name: ZOO_SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx512M"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data/zookeeper/data"
            name: zookeeper-datadir-pvc-1 
      volumes:
        - name: zookeeper-datadir-pvc-1 
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-1
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: zookeeper2
  namespace: zookeeper
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "2"
    spec:
      containers:
        - name: server
          image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1 
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "2"
            - name: ZOO_SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx512M"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data/zookeeper/data"
            name: zookeeper-datadir-pvc-2 
      volumes:
        - name: zookeeper-datadir-pvc-2 
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-2
---
kind: Deployment
apiVersion: apps/v1
metadata:
  name: zookeeper3
  namespace: zookeeper
spec:
  replicas: 1
  selector:
    matchLabels:
      app: zookeeper
  template:
    metadata:
      labels:
        app: zookeeper
        server-id: "3"
    spec:
      containers:
        - name: server
          image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1
          imagePullPolicy: Always
          env:
            - name: ZOO_MY_ID
              value: "3"
            - name: ZOO_SERVERS
              value: "zookeeper1,zookeeper2,zookeeper3"
            - name: JVMFLAGS
              value: "-Xmx512M"
          ports:
            - containerPort: 2181
            - containerPort: 2888
            - containerPort: 3888
          volumeMounts:
          - mountPath: "/data/zookeeper/data"
            name: zookeeper-datadir-pvc-3 
      volumes:
        - name: zookeeper-datadir-pvc-3 
          persistentVolumeClaim:
            claimName: zookeeper-datadir-pvc-3

创建资源

# kubectl apply -f .

namespace/zookeeper created
persistentvolume/zookeeper-datadir-pv-1 created
persistentvolume/zookeeper-datadir-pv-2 created
persistentvolume/zookeeper-datadir-pv-3 created
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persistentvolumeclaim/zookeeper-datadir-pvc-3 created
service/zookeeper1 created
service/zookeeper2 created
service/zookeeper3 created
deployment.apps/zookeeper1 created
deployment.apps/zookeeper2 created
deployment.apps/zookeeper3 created

查看资源

查看zookeeper pod

# kubectl get pod -n zookeeper

NAME                          READY   STATUS    RESTARTS   AGE
zookeeper1-6848b5c969-st2mm   1/1     Running   0          92s
zookeeper2-56879d5d5f-m5g56   1/1     Running   0          92s
zookeeper3-5bfc89596f-n4znb   1/1     Running   0          92s

查看zookeeper ns

# kubectl get ns 

NAME                   STATUS   AGE
default                Active   26d
kube-node-lease        Active   26d
kube-public            Active   26d
kube-system            Active   26d
kubernetes-dashboard   Active   25d
zookeeper              Active   28s

查看zookeeper svc

# kubectl get svc -n zookeeper -o wide

NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)                                        AGE   SELECTOR
zookeeper1   NodePort    10.100.229.141   <none>        2181:42181/TCP,2888:43932/TCP,3888:49441/TCP   16m   app=zookeeper,server-id=1
zookeeper2   NodePort    10.100.38.155    <none>        2181:42182/TCP,2888:49756/TCP,3888:38345/TCP   16m   app=zookeeper,server-id=2
zookeeper3   NodePort    10.100.198.155   <none>        2181:42183/TCP,2888:44017/TCP,3888:32672/TCP   16m   app=zookeeper,server-id=3

 

验证zookeeper集群配置

# kubectl exec -it zookeeper1-6848b5c969-st2mm -n zookeeper -- cat /usr/local/zookeeper/conf/zoo.cfg

tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/datalog
autopurge.purgeInterval=1
autopurge.snapRetainCount=3
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
clientPort=2181
quorumListenOnAllIPs=true

server.1=zookeeper1:2888:3888
server.2=zookeeper2:2888:3888
server.3=zookeeper3:2888:3888

查看zookeeper集群状态

zookeeper1

# kubectl exec -it zookeeper1-6848b5c969-st2mm -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

zookeeper2

# kubectl exec -it zookeeper2-56879d5d5f-m5g56 -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower

zookeeper3

# kubectl exec -it zookeeper3-5bfc89596f-n4znb -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status

ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader

查看数据目录信息

# tree /data/k8s-data/zookeeper/

/data/k8s-data/zookeeper/
├── zookeeper-datadir-1
│   ├── myid
│   └── version-2
│       ├── acceptedEpoch
│       ├── currentEpoch
│       └── snapshot.0
├── zookeeper-datadir-2
│   ├── myid
│   └── version-2
│       ├── acceptedEpoch
│       ├── currentEpoch
│       └── snapshot.0
└── zookeeper-datadir-3
    ├── myid
    └── version-2
        ├── acceptedEpoch
        ├── currentEpoch
        └── snapshot.0

6 directories, 12 files

访问zookeeper集群

# kubectl run zk-client --image=192.168.174.120/baseimages/zookeeker:3.7.0 -it --rm --restart=Never -n zookeeper -- /usr/local/zookeeper/bin/zkCli.sh -server zookeeper1.zookeeper

Connecting to zookeeper1.zookeeper
2021-12-15 10:33:41,408 [myid:] - INFO  [main:Environment@98] - Client environment:zookeeper.version=3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
2021-12-15 10:33:41,411 [myid:] - INFO  [main:Environment@98] - Client environment:host.name=ceph-node01
2021-12-15 10:33:41,411 [myid:] - INFO  [main:Environment@98] - Client environment:java.version=17.0.1
2021-12-15 10:33:41,433 [myid:] - INFO  [main:Environment@98] - Client environment:java.vendor=Oracle Corporation
2021-12-15 10:33:41,433 [myid:] - INFO  [main:Environment@98] - Client environment:java.home=/usr/local/jdk-17.0.1
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:java.class.path=/usr/local/zookeeper/bin/../zookeeper-server/target/classes:/usr/local/zookeeper/bin/../build/classes:/usr/local/zookeeper/bin/../zookeeper-server/target/lib/*.jar:/usr/local/zookeeper/bin/../build/lib/*.jar:/usr/local/zookeeper/bin/../lib/zookeeper-prometheus-metrics-3.7.0.jar:/usr/local/zookeeper/bin/../lib/zookeeper-jute-3.7.0.jar:/usr/local/zookeeper/bin/../lib/zookeeper-3.7.0.jar:/usr/local/zookeeper/bin/../lib/snappy-java-1.1.7.7.jar:/usr/local/zookeeper/bin/../lib/slf4j-log4j12-1.7.30.jar:/usr/local/zookeeper/bin/../lib/slf4j-api-1.7.30.jar:/usr/local/zookeeper/bin/../lib/simpleclient_servlet-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient_hotspot-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient_common-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient-0.9.0.jar:/usr/local/zookeeper/bin/../lib/netty-transport-native-unix-common-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-transport-native-epoll-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-transport-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-resolver-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-handler-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-common-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-codec-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-buffer-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/metrics-core-4.1.12.1.jar:/usr/local/zookeeper/bin/../lib/log4j-1.2.17.jar:/usr/local/zookeeper/bin/../lib/jline-2.14.6.jar:/usr/local/zookeeper/bin/../lib/jetty-util-ajax-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-util-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-servlet-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-server-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-security-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-io-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-http-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/javax.servlet-api-3.1.0.jar:/usr/local/zookeeper/bin/../lib/jackson-databind-2.10.5.1.jar:/usr/local/zookeeper/bin/../lib/jackson-core-2.10.5.jar:/usr/local/zookeeper/bin/../lib/jackson-annotations-2.10.5.jar:/usr/local/zookeeper/bin/../lib/commons-cli-1.4.jar:/usr/local/zookeeper/bin/../lib/audience-annotations-0.12.0.jar:/usr/local/zookeeper/bin/../zookeeper-*.jar:/usr/local/zookeeper/bin/../zookeeper-server/src/main/resources/lib/*.jar:/usr/local/zookeeper/bin/../conf:.:/usr/local/jdk/lib:/usr/local/jdk/jre/lib:/usr/local/jdk/lib/tools.jar
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:java.io.tmpdir=/tmp
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:java.compiler=<NA>
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:os.name=Linux
2021-12-15 10:33:41,434 [myid:] - INFO  [main:Environment@98] - Client environment:os.arch=amd64
2021-12-15 10:33:41,435 [myid:] - INFO  [main:Environment@98] - Client environment:os.version=5.4.0-91-generic
2021-12-15 10:33:41,435 [myid:] - INFO  [main:Environment@98] - Client environment:user.name=root
2021-12-15 10:33:41,435 [myid:] - INFO  [main:Environment@98] - Client environment:user.home=/root
2021-12-15 10:33:41,435 [myid:] - INFO  [main:Environment@98] - Client environment:user.dir=/root
2021-12-15 10:33:41,435 [myid:] - INFO  [main:Environment@98] - Client environment:os.memory.free=27MB
2021-12-15 10:33:41,436 [myid:] - INFO  [main:Environment@98] - Client environment:os.memory.max=256MB
2021-12-15 10:33:41,436 [myid:] - INFO  [main:Environment@98] - Client environment:os.memory.total=34MB
2021-12-15 10:33:41,440 [myid:] - INFO  [main:ZooKeeper@637] - Initiating client connection, connectString=192.168.174.106:42181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@6fadae5d
2021-12-15 10:33:41,472 [myid:] - INFO  [main:X509Util@77] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation
2021-12-15 10:33:41,489 [myid:] - INFO  [main:ClientCnxnSocket@239] - jute.maxbuffer value is 1048575 Bytes
2021-12-15 10:33:41,514 [myid:] - INFO  [main:ClientCnxn@1726] - zookeeper.request.timeout value is 0. feature enabled=false
Welcome to ZooKeeper!
2021-12-15 10:33:41,561 [myid:zookeeper1.zookeeper:2181] - INFO  [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1171] - Opening socket connection to server zookeeper1.zookeeper/10.100.43.156:2181.
2021-12-15 10:33:41,562 [myid:zookeeper1.zookeeper:2181] - INFO  [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1173] - SASL config status: Will not attempt to authenticate using SASL (unknown error)
JLine support is enabled
2021-12-15 10:33:41,629 [myid:zookeeper1.zookeeper:2181] - INFO  [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1005] - Socket connection established, initiating session, client: /172.20.89.132:41678, server: zookeeper1.zookeeper/10.100.43.156:2181
2021-12-15 10:33:41,658 [myid:zookeeper1.zookeeper:2181] - INFO  [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1438] - Session establishment complete on server zookeeper1.zookeeper/10.100.43.156:2181, session id = 0x10000cc4e8d0001, negotiated timeout = 30000

WATCHER::

WatchedEvent state:SyncConnected type:None path:null
[zk: zookeeper1.zookeeper(CONNECTED) 0]