zookeeper-ns.yaml
apiVersion: v1
kind: Namespace
metadata:
name: zookeeper
zookeeper-persistentvolume.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: zookeeper-datadir-pv-1
namespace: zookeeper
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.174.102
path: /data/k8s-data//zookeeper/zookeeper-datadir-1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: zookeeper-datadir-pv-2
namespace: zookeeper
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.174.102
path: /data/k8s-data//zookeeper/zookeeper-datadir-1
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: zookeeper-datadir-pv-3
namespace: zookeeper
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteOnce
nfs:
server: 192.168.174.102
path: /data/k8s-data//zookeeper/zookeeper-datadir-1
zookeeper-persistentvolumeclaim.yaml
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: zookeeper-datadir-pvc-1
namespace: zookeeper
spec:
accessModes:
- ReadWriteOnce
volumeName: zookeeper-datadir-pv-1
resources:
requests:
storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: zookeeper-datadir-pvc-2
namespace: zookeeper
spec:
accessModes:
- ReadWriteOnce
volumeName: zookeeper-datadir-pv-2
resources:
requests:
storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: zookeeper-datadir-pvc-3
namespace: zookeeper
spec:
accessModes:
- ReadWriteOnce
volumeName: zookeeper-datadir-pv-3
resources:
requests:
storage: 5Gi
zookeeper.yaml
apiVersion: v1
kind: Service
metadata:
name: zookeeper1
namespace: zookeeper
spec:
type: NodePort
ports:
- name: client
port: 2181
nodePort: 42181
- name: followers
port: 2888
- name: election
port: 3888
selector:
app: zookeeper
server-id: "1"
---
apiVersion: v1
kind: Service
metadata:
name: zookeeper2
namespace: zookeeper
spec:
type: NodePort
ports:
- name: client
port: 2181
nodePort: 42182
- name: followers
port: 2888
- name: election
port: 3888
selector:
app: zookeeper
server-id: "2"
---
apiVersion: v1
kind: Service
metadata:
name: zookeeper3
namespace: zookeeper
spec:
type: NodePort
ports:
- name: client
port: 2181
nodePort: 42183
- name: followers
port: 2888
- name: election
port: 3888
selector:
app: zookeeper
server-id: "3"
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper1
namespace: zookeeper
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
server-id: "1"
spec:
containers:
- name: server
image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1
imagePullPolicy: Always
env:
- name: ZOO_MY_ID
value: "1"
- name: ZOO_SERVERS
value: "zookeeper1,zookeeper2,zookeeper3"
- name: JVMFLAGS
value: "-Xmx512M"
ports:
- containerPort: 2181
- containerPort: 2888
- containerPort: 3888
volumeMounts:
- mountPath: "/data/zookeeper/data"
name: zookeeper-datadir-pvc-1
volumes:
- name: zookeeper-datadir-pvc-1
persistentVolumeClaim:
claimName: zookeeper-datadir-pvc-1
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper2
namespace: zookeeper
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
server-id: "2"
spec:
containers:
- name: server
image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1
imagePullPolicy: Always
env:
- name: ZOO_MY_ID
value: "2"
- name: ZOO_SERVERS
value: "zookeeper1,zookeeper2,zookeeper3"
- name: JVMFLAGS
value: "-Xmx512M"
ports:
- containerPort: 2181
- containerPort: 2888
- containerPort: 3888
volumeMounts:
- mountPath: "/data/zookeeper/data"
name: zookeeper-datadir-pvc-2
volumes:
- name: zookeeper-datadir-pvc-2
persistentVolumeClaim:
claimName: zookeeper-datadir-pvc-2
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: zookeeper3
namespace: zookeeper
spec:
replicas: 1
selector:
matchLabels:
app: zookeeper
template:
metadata:
labels:
app: zookeeper
server-id: "3"
spec:
containers:
- name: server
image: uhub.service.ucloud.cn/wgs-test/zookeeker:3.8.1
imagePullPolicy: Always
env:
- name: ZOO_MY_ID
value: "3"
- name: ZOO_SERVERS
value: "zookeeper1,zookeeper2,zookeeper3"
- name: JVMFLAGS
value: "-Xmx512M"
ports:
- containerPort: 2181
- containerPort: 2888
- containerPort: 3888
volumeMounts:
- mountPath: "/data/zookeeper/data"
name: zookeeper-datadir-pvc-3
volumes:
- name: zookeeper-datadir-pvc-3
persistentVolumeClaim:
claimName: zookeeper-datadir-pvc-3
创建资源
# kubectl apply -f .
namespace/zookeeper created
persistentvolume/zookeeper-datadir-pv-1 created
persistentvolume/zookeeper-datadir-pv-2 created
persistentvolume/zookeeper-datadir-pv-3 created
persistentvolumeclaim/zookeeper-datadir-pvc-1 created
persistentvolumeclaim/zookeeper-datadir-pvc-2 created
persistentvolumeclaim/zookeeper-datadir-pvc-3 created
service/zookeeper1 created
service/zookeeper2 created
service/zookeeper3 created
deployment.apps/zookeeper1 created
deployment.apps/zookeeper2 created
deployment.apps/zookeeper3 created
查看资源
查看zookeeper pod
# kubectl get pod -n zookeeper
NAME READY STATUS RESTARTS AGE
zookeeper1-6848b5c969-st2mm 1/1 Running 0 92s
zookeeper2-56879d5d5f-m5g56 1/1 Running 0 92s
zookeeper3-5bfc89596f-n4znb 1/1 Running 0 92s
查看zookeeper ns
# kubectl get ns
NAME STATUS AGE
default Active 26d
kube-node-lease Active 26d
kube-public Active 26d
kube-system Active 26d
kubernetes-dashboard Active 25d
zookeeper Active 28s
查看zookeeper svc
# kubectl get svc -n zookeeper -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
zookeeper1 NodePort 10.100.229.141 <none> 2181:42181/TCP,2888:43932/TCP,3888:49441/TCP 16m app=zookeeper,server-id=1
zookeeper2 NodePort 10.100.38.155 <none> 2181:42182/TCP,2888:49756/TCP,3888:38345/TCP 16m app=zookeeper,server-id=2
zookeeper3 NodePort 10.100.198.155 <none> 2181:42183/TCP,2888:44017/TCP,3888:32672/TCP 16m app=zookeeper,server-id=3
验证zookeeper集群配置
# kubectl exec -it zookeeper1-6848b5c969-st2mm -n zookeeper -- cat /usr/local/zookeeper/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zookeeper/data
dataLogDir=/data/zookeeper/datalog
autopurge.purgeInterval=1
autopurge.snapRetainCount=3
maxClientCnxns=60
standaloneEnabled=true
admin.enableServer=true
clientPort=2181
quorumListenOnAllIPs=true
server.1=zookeeper1:2888:3888
server.2=zookeeper2:2888:3888
server.3=zookeeper3:2888:3888
查看zookeeper集群状态
zookeeper1
# kubectl exec -it zookeeper1-6848b5c969-st2mm -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
zookeeper2
# kubectl exec -it zookeeper2-56879d5d5f-m5g56 -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: follower
zookeeper3
# kubectl exec -it zookeeper3-5bfc89596f-n4znb -n zookeeper -- /usr/local/zookeeper/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost. Client SSL: false.
Mode: leader
查看数据目录信息
# tree /data/k8s-data/zookeeper/
/data/k8s-data/zookeeper/
├── zookeeper-datadir-1
│ ├── myid
│ └── version-2
│ ├── acceptedEpoch
│ ├── currentEpoch
│ └── snapshot.0
├── zookeeper-datadir-2
│ ├── myid
│ └── version-2
│ ├── acceptedEpoch
│ ├── currentEpoch
│ └── snapshot.0
└── zookeeper-datadir-3
├── myid
└── version-2
├── acceptedEpoch
├── currentEpoch
└── snapshot.0
6 directories, 12 files
访问zookeeper集群
# kubectl run zk-client --image=192.168.174.120/baseimages/zookeeker:3.7.0 -it --rm --restart=Never -n zookeeper -- /usr/local/zookeeper/bin/zkCli.sh -server zookeeper1.zookeeper
Connecting to zookeeper1.zookeeper
2021-12-15 10:33:41,408 [myid:] - INFO [main:Environment@98] - Client environment:zookeeper.version=3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC
2021-12-15 10:33:41,411 [myid:] - INFO [main:Environment@98] - Client environment:host.name=ceph-node01
2021-12-15 10:33:41,411 [myid:] - INFO [main:Environment@98] - Client environment:java.version=17.0.1
2021-12-15 10:33:41,433 [myid:] - INFO [main:Environment@98] - Client environment:java.vendor=Oracle Corporation
2021-12-15 10:33:41,433 [myid:] - INFO [main:Environment@98] - Client environment:java.home=/usr/local/jdk-17.0.1
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:java.class.path=/usr/local/zookeeper/bin/../zookeeper-server/target/classes:/usr/local/zookeeper/bin/../build/classes:/usr/local/zookeeper/bin/../zookeeper-server/target/lib/*.jar:/usr/local/zookeeper/bin/../build/lib/*.jar:/usr/local/zookeeper/bin/../lib/zookeeper-prometheus-metrics-3.7.0.jar:/usr/local/zookeeper/bin/../lib/zookeeper-jute-3.7.0.jar:/usr/local/zookeeper/bin/../lib/zookeeper-3.7.0.jar:/usr/local/zookeeper/bin/../lib/snappy-java-1.1.7.7.jar:/usr/local/zookeeper/bin/../lib/slf4j-log4j12-1.7.30.jar:/usr/local/zookeeper/bin/../lib/slf4j-api-1.7.30.jar:/usr/local/zookeeper/bin/../lib/simpleclient_servlet-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient_hotspot-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient_common-0.9.0.jar:/usr/local/zookeeper/bin/../lib/simpleclient-0.9.0.jar:/usr/local/zookeeper/bin/../lib/netty-transport-native-unix-common-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-transport-native-epoll-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-transport-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-resolver-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-handler-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-common-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-codec-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/netty-buffer-4.1.59.Final.jar:/usr/local/zookeeper/bin/../lib/metrics-core-4.1.12.1.jar:/usr/local/zookeeper/bin/../lib/log4j-1.2.17.jar:/usr/local/zookeeper/bin/../lib/jline-2.14.6.jar:/usr/local/zookeeper/bin/../lib/jetty-util-ajax-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-util-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-servlet-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-server-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-security-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-io-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/jetty-http-9.4.38.v20210224.jar:/usr/local/zookeeper/bin/../lib/javax.servlet-api-3.1.0.jar:/usr/local/zookeeper/bin/../lib/jackson-databind-2.10.5.1.jar:/usr/local/zookeeper/bin/../lib/jackson-core-2.10.5.jar:/usr/local/zookeeper/bin/../lib/jackson-annotations-2.10.5.jar:/usr/local/zookeeper/bin/../lib/commons-cli-1.4.jar:/usr/local/zookeeper/bin/../lib/audience-annotations-0.12.0.jar:/usr/local/zookeeper/bin/../zookeeper-*.jar:/usr/local/zookeeper/bin/../zookeeper-server/src/main/resources/lib/*.jar:/usr/local/zookeeper/bin/../conf:.:/usr/local/jdk/lib:/usr/local/jdk/jre/lib:/usr/local/jdk/lib/tools.jar
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:java.io.tmpdir=/tmp
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:java.compiler=<NA>
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:os.name=Linux
2021-12-15 10:33:41,434 [myid:] - INFO [main:Environment@98] - Client environment:os.arch=amd64
2021-12-15 10:33:41,435 [myid:] - INFO [main:Environment@98] - Client environment:os.version=5.4.0-91-generic
2021-12-15 10:33:41,435 [myid:] - INFO [main:Environment@98] - Client environment:user.name=root
2021-12-15 10:33:41,435 [myid:] - INFO [main:Environment@98] - Client environment:user.home=/root
2021-12-15 10:33:41,435 [myid:] - INFO [main:Environment@98] - Client environment:user.dir=/root
2021-12-15 10:33:41,435 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.free=27MB
2021-12-15 10:33:41,436 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.max=256MB
2021-12-15 10:33:41,436 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.total=34MB
2021-12-15 10:33:41,440 [myid:] - INFO [main:ZooKeeper@637] - Initiating client connection, connectString=192.168.174.106:42181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@6fadae5d
2021-12-15 10:33:41,472 [myid:] - INFO [main:X509Util@77] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation
2021-12-15 10:33:41,489 [myid:] - INFO [main:ClientCnxnSocket@239] - jute.maxbuffer value is 1048575 Bytes
2021-12-15 10:33:41,514 [myid:] - INFO [main:ClientCnxn@1726] - zookeeper.request.timeout value is 0. feature enabled=false
Welcome to ZooKeeper!
2021-12-15 10:33:41,561 [myid:zookeeper1.zookeeper:2181] - INFO [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1171] - Opening socket connection to server zookeeper1.zookeeper/10.100.43.156:2181.
2021-12-15 10:33:41,562 [myid:zookeeper1.zookeeper:2181] - INFO [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1173] - SASL config status: Will not attempt to authenticate using SASL (unknown error)
JLine support is enabled
2021-12-15 10:33:41,629 [myid:zookeeper1.zookeeper:2181] - INFO [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1005] - Socket connection established, initiating session, client: /172.20.89.132:41678, server: zookeeper1.zookeeper/10.100.43.156:2181
2021-12-15 10:33:41,658 [myid:zookeeper1.zookeeper:2181] - INFO [main-SendThread(zookeeper1.zookeeper:2181):ClientCnxn$SendThread@1438] - Session establishment complete on server zookeeper1.zookeeper/10.100.43.156:2181, session id = 0x10000cc4e8d0001, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
[zk: zookeeper1.zookeeper(CONNECTED) 0]
