部署 Statefulset 服务 Zookeeper
部署 Statefulset 服务 Zookeeper
创建 Zookeeper 镜像
下载 Zookeeper 二进制包
mkdir zk
cd zk/
wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.1/apache-zookeeper-3.8.1-bin.tar.gz
编写 zk 配置脚本
cat <<EOF> zkGenConfig.sh
#!/usr/bin/env bash
ZK_USER=${ZK_USER:-"zookeeper"}
ZK_LOG_LEVEL=${ZK_LOG_LEVEL:-"INFO"}
ZK_DATA_DIR=${ZK_DATA_DIR:-"/data/zookeeper/data_dir"} #change
ZK_DATA_LOG_DIR=${ZK_DATA_LOG_DIR:-"/data/zookeeper/log_dir"} #change
ZK_LOG_DIR=${ZK_LOG_DIR:-"/data/zookeeper/logs"} #change
ZK_CONF_DIR=${ZK_CONF_DIR:-"/opt/zookeeper/conf"}
ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
ZK_SERVER_PORT=${ZK_SERVER_PORT:-2888}
ZK_ELECTION_PORT=${ZK_ELECTION_PORT:-3888}
ZK_TICK_TIME=${ZK_TICK_TIME:-2000}
ZK_INIT_LIMIT=${ZK_INIT_LIMIT:-10}
ZK_SYNC_LIMIT=${ZK_SYNC_LIMIT:-5}
ZK_HEAP_SIZE=${ZK_HEAP_SIZE:-2G}
ZK_MAX_CLIENT_CNXNS=${ZK_MAX_CLIENT_CNXNS:-1000} #change
ZK_MIN_SESSION_TIMEOUT=${ZK_MIN_SESSION_TIMEOUT:- $((ZK_TICK_TIME*2))}
ZK_MAX_SESSION_TIMEOUT=${ZK_MAX_SESSION_TIMEOUT:- $((ZK_TICK_TIME*20))}
ZK_SNAP_RETAIN_COUNT=${ZK_SNAP_RETAIN_COUNT:-3}
ZK_PURGE_INTERVAL=${ZK_PURGE_INTERVAL:-0}
ID_FILE="$ZK_DATA_DIR/myid"
ZK_CONFIG_FILE="$ZK_CONF_DIR/zoo.cfg"
LOGGER_PROPS_FILE="$ZK_CONF_DIR/log4j.properties"
JAVA_ENV_FILE="$ZK_CONF_DIR/java.env"
HOST=`hostname -s`
DOMAIN=`hostname -d`
function print_servers() {
for (( i=1; i<=$ZK_REPLICAS; i++ ))
do
echo "server.$i=$NAME-$((i-1)).$DOMAIN:$ZK_SERVER_PORT:$ZK_ELECTION_PORT"
done
}
function validate_env() {
echo "Validating environment"
if [ -z $ZK_REPLICAS ]; then
echo "ZK_REPLICAS is a mandatory environment variable"
exit 1
fi
if [[ $HOST =~ (.*)-([0-9]+)$ ]]; then
NAME=${BASH_REMATCH[1]}
ORD=${BASH_REMATCH[2]}
else
echo "Failed to extract ordinal from hostname $HOST"
exit 1
fi
MY_ID=$((ORD+1))
echo "ZK_REPLICAS=$ZK_REPLICAS"
echo "MY_ID=$MY_ID"
echo "ZK_LOG_LEVEL=$ZK_LOG_LEVEL"
echo "ZK_DATA_DIR=$ZK_DATA_DIR"
echo "ZK_DATA_LOG_DIR=$ZK_DATA_LOG_DIR"
echo "ZK_LOG_DIR=$ZK_LOG_DIR"
echo "ZK_CLIENT_PORT=$ZK_CLIENT_PORT"
echo "ZK_SERVER_PORT=$ZK_SERVER_PORT"
echo "ZK_ELECTION_PORT=$ZK_ELECTION_PORT"
echo "ZK_TICK_TIME=$ZK_TICK_TIME"
echo "ZK_INIT_LIMIT=$ZK_INIT_LIMIT"
echo "ZK_SYNC_LIMIT=$ZK_SYNC_LIMIT"
echo "ZK_MAX_CLIENT_CNXNS=$ZK_MAX_CLIENT_CNXNS"
echo "ZK_MIN_SESSION_TIMEOUT=$ZK_MIN_SESSION_TIMEOUT"
echo "ZK_MAX_SESSION_TIMEOUT=$ZK_MAX_SESSION_TIMEOUT"
echo "ZK_HEAP_SIZE=$ZK_HEAP_SIZE"
echo "ZK_SNAP_RETAIN_COUNT=$ZK_SNAP_RETAIN_COUNT"
echo "ZK_PURGE_INTERVAL=$ZK_PURGE_INTERVAL"
echo "ENSEMBLE"
print_servers
echo "Environment validation successful"
}
function create_config() {
rm -f $ZK_CONFIG_FILE
echo "Creating ZooKeeper configuration"
echo "#This file was autogenerated by k8szk DO NOT EDIT" >> $ZK_CONFIG_FILE
echo "clientPort=$ZK_CLIENT_PORT" >> $ZK_CONFIG_FILE
echo "dataDir=$ZK_DATA_DIR" >> $ZK_CONFIG_FILE
echo "dataLogDir=$ZK_DATA_LOG_DIR" >> $ZK_CONFIG_FILE
echo "tickTime=$ZK_TICK_TIME" >> $ZK_CONFIG_FILE
echo "initLimit=$ZK_INIT_LIMIT" >> $ZK_CONFIG_FILE
echo "syncLimit=$ZK_SYNC_LIMIT" >> $ZK_CONFIG_FILE
echo "maxClientCnxns=$ZK_MAX_CLIENT_CNXNS" >> $ZK_CONFIG_FILE
echo "minSessionTimeout=$ZK_MIN_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
echo "maxSessionTimeout=$ZK_MAX_SESSION_TIMEOUT" >> $ZK_CONFIG_FILE
echo "autopurge.snapRetainCount=$ZK_SNAP_RETAIN_COUNT" >> $ZK_CONFIG_FILE
echo "autopurge.purgeInterval=$ZK_PURGE_INTERVAL" >> $ZK_CONFIG_FILE
echo "4lw.commands.whitelist=*" >> $ZK_CONFIG_FILE
echo "quorumListenOnAllIPs=true" >> $ZK_CONFIG_FILE
if [ $ZK_REPLICAS -gt 1 ]; then
print_servers >> $ZK_CONFIG_FILE
fi
echo "Wrote ZooKeeper configuration file to $ZK_CONFIG_FILE"
}
function create_data_dirs() {
echo "Creating ZooKeeper data directories and setting permissions"
if [ ! -d $ZK_DATA_DIR ]; then
mkdir -p $ZK_DATA_DIR
chown -R $ZK_USER:$ZK_USER $ZK_DATA_DIR
fi
if [ ! -d $ZK_DATA_LOG_DIR ]; then
mkdir -p $ZK_DATA_LOG_DIR
chown -R $ZK_USER:$ZK_USER $ZK_DATA_LOG_DIR
fi
if [ ! -d $ZK_LOG_DIR ]; then
mkdir -p $ZK_LOG_DIR
chown -R $ZK_USER:$ZK_USER $ZK_LOG_DIR
fi
if [ ! -f $ID_FILE ]; then
echo $MY_ID >> $ID_FILE
fi
echo "Created ZooKeeper data directories"
}
function create_log_props () {
rm -f $LOGGER_PROPS_FILE
echo "Creating ZooKeeper log4j configuration"
echo "zookeeper.root.logger=CONSOLE" >> $LOGGER_PROPS_FILE
echo "zookeeper.console.threshold="$ZK_LOG_LEVEL >> $LOGGER_PROPS_FILE
echo "log4j.rootLogger=\${zookeeper.root.logger}" >> $LOGGER_PROPS_FILE
echo "log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender" >> $LOGGER_PROPS_FILE
echo "log4j.appender.CONSOLE.Threshold=\${zookeeper.console.threshold}" >> $LOGGER_PROPS_FILE
echo "log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout" >> $LOGGER_PROPS_FILE
echo "log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n" >> $LOGGER_PROPS_FILE
echo "Wrote log4j configuration to $LOGGER_PROPS_FILE"
}
function create_java_env() {
rm -f $JAVA_ENV_FILE
echo "Creating JVM configuration file"
echo "ZOO_LOG_DIR=$ZK_LOG_DIR" >> $JAVA_ENV_FILE
echo "JVMFLAGS=\"-Xmx$ZK_HEAP_SIZE -Xms$ZK_HEAP_SIZE\"" >> $JAVA_ENV_FILE
echo "Wrote JVM configuration to $JAVA_ENV_FILE"
}
validate_env && create_config && create_log_props && create_data_dirs && create_java_env
EOF
编写 zk 服务的健康检查脚本
cat <<EOF> zkOk.sh
#!/usr/bin/env bash
# zkOk.sh uses the ruok ZooKeeper four letter work to determine if the instance
# is health. The $? variable will be set to 0 if server responds that it is
# healthy, or 1 if the server fails to respond.
ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
OK=$(echo ruok | nc 127.0.0.1 $ZK_CLIENT_PORT)
if [ "$OK" == "imok" ]; then
exit 0
else
exit 1
fi
EOF
编写获取 zk 配置参数的脚本
cat <<EOF> zkMetrics.sh
#!/usr/bin/env bash
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
ZK_CLIENT_PORT=${ZK_CLIENT_PORT:-2181}
echo mntr | nc localhost $ZK_CLIENT_PORT >& 1
EOF
编写 Dockerfile 文件
cat <<EOF> Dockerfile
FROM ubuntu:20.04
MAINTAINER liumanlin@zhizhangyi.com
#defaine env args
ENV ZK_USER=zookeeper \
ZK_DATA_DIR=/data/zookeeper/data_dir \
ZK_DATA_LOG_DIR=/data/zookeeper/log_dir \
ZK_LOG_DIR=/data/zookeeper/logs \
JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \
ZK_DIST=apache-zookeeper-3.8.1-bin
#copy zk package
ADD $ZK_DIST.tar.gz /opt
#install java nc tools and clean cache
RUN set -x \
&& apt-get update \
&& apt-get install -y openjdk-8-jre-headless netcat-openbsd \
&& ln -s /opt/$ZK_DIST /opt/zookeeper \
&& rm -rf /var/lib/apt/lists/*
# Copy configuration generator script to bin
COPY zkGenConfig.sh zkOk.sh zkMetrics.sh /opt/zookeeper/bin/
# Create a user for the zookeeper process and configure file system ownership
# for necessary directories and symlink the distribution as a user executable
RUN set -x \
&& useradd $ZK_USER \
&& [ `id -u $ZK_USER` -eq 1000 ] \
&& [ `id -g $ZK_USER` -eq 1000 ] \
&& chmod +x /opt/zookeeper/bin/zkGenConfig.sh \
&& chmod +x /opt/zookeeper/bin/zkOk.sh \
&& chmod +x /opt/zookeeper/bin/zkMetrics.sh \
&& mkdir -p $ZK_DATA_DIR $ZK_DATA_LOG_DIR $ZK_LOG_DIR /usr/share/zookeeper /usr/etc/ \
&& chown -R "$ZK_USER:$ZK_USER" /opt/$ZK_DIST $ZK_DATA_DIR $ZK_LOG_DIR $ZK_DATA_LOG_DIR \
&& ln -s /opt/zookeeper/conf/ /usr/etc/zookeeper \
&& ln -s /opt/zookeeper/bin/* /usr/bin \
&& ln -s /opt/zookeeper/lib/* /usr/share/zookeeper
EOF
打包构建
docker build -t harbor.zhizhangyi.com/k8s-middle/zzy-zookeeper:v3.8.1 .
推动至镜像仓库
docker push harbor.zhizhangyi.com/k8s-middle/zzy-zookeeper:v3.8.1
部署 Statefulset 服务
编写 zk 的配置文件
cat <<EOF> zk-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: zk-cm
namespace: zzy-mbs
data:
jvm.heap: "1G"
tick: "2000"
init: "10"
sync: "5"
client.cnxns: "1000"
snap.retain: "3"
purge.interval: "0"
EOF
编写 zk 的 pdb 文件
保证集群最少可用的节点数。
cat <<EOF> zk-pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: zk-pdb
namespace: zzy-mbs
spec:
selector:
matchLabels:
app: zk
minAvailable: 2
EOF
编写 zk 的 svc 文件
cat <<EOF>zk-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: zk-cs
namespace: zzy-mbs
labels:
app: zk
spec:
selector:
app: zk
ports:
- port: 2181
name: client
---
apiVersion: v1
kind: Service
metadata:
name: zk-svc
namespace: zzy-mbs
labels:
app: zk-svc
spec:
ports:
- port: 2888
name: server
- port: 3888
name: leader-election
clusterIP: None
selector:
app: zk
EOF
编写 zk 的 sts 文件
cat <<EOF> zk-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: zk
namespace: zzy-mbs
spec:
selector:
matchLabels:
app: zk
serviceName: zk-svc
replicas: 3
template:
metadata:
labels:
app: zk
spec:
imagePullSecrets:
- name: harbor
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: "app"
operator: In
values:
- zk
topologyKey: "kubernetes.io/hostname"
containers:
- name: zzy-zk
imagePullPolicy: Always
image: harbor.zhizhangyi.com/k8s-middle/zzy-zookeeper:v3.8.1
resources:
requests:
memory: "1Gi"
cpu: "500m"
ports:
- containerPort: 2181
name: client
- containerPort: 2888
name: server
- containerPort: 3888
name: leader-election
env:
- name: ZK_REPLICAS
value: "3"
- name: ZK_HEAP_SIZE
valueFrom:
configMapKeyRef:
name: zk-cm
key: jvm.heap
- name: ZK_TICK_TIME
valueFrom:
configMapKeyRef:
name: zk-cm
key: tick
- name: ZK_INIT_LIMIT
valueFrom:
configMapKeyRef:
name: zk-cm
key: init
- name: ZK_SYNC_LIMIT
valueFrom:
configMapKeyRef:
name: zk-cm
key: tick
- name: ZK_MAX_CLIENT_CNXNS
valueFrom:
configMapKeyRef:
name: zk-cm
key: client.cnxns
- name: ZK_SNAP_RETAIN_COUNT
valueFrom:
configMapKeyRef:
name: zk-cm
key: snap.retain
- name: ZK_PURGE_INTERVAL
valueFrom:
configMapKeyRef:
name: zk-cm
key: purge.interval
- name: ZK_CLIENT_PORT
value: "2181"
- name: ZK_SERVER_PORT
value: "2888"
- name: ZK_ELECTION_PORT
value: "3888"
command:
- sh
- -c
- zkGenConfig.sh && zkServer.sh start-foreground
readinessProbe:
exec:
command:
- "zkOk.sh"
initialDelaySeconds: 10
timeoutSeconds: 5
livenessProbe:
exec:
command:
- "zkOk.sh"
initialDelaySeconds: 10
timeoutSeconds: 5
volumeMounts:
- name: datadir
mountPath: /data/zookeeper
securityContext:
runAsUser: 1000
fsGroup: 1000
volumeClaimTemplates:
- metadata:
name: datadir
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: "nfs-storage"
resources:
requests:
storage: 10Gi
EOF
依次执行 yaml 文件
kubectl apply -f zk-cm.yaml
kubectl apply -f zk-pdb.yaml
kubectl apply -f zk-svc.yaml
kubectl apply -f zk-sts.yaml
验证
## 查询 pod
kubectl get pod -n zzy-mbs
## 查看状态
for i in 0 1 2; do kubectl exec -it zk-$i zkServer.sh status -n zzy-mbs; done
## 查看版本
for i in 0 1 2; do kubectl exec -it zk-$i zkServer.sh version -n zzy-mbs; done
参考
- Zookeeper 创建镜像：https://blog.csdn.net/weixin_44729138/article/details/120326359
- 部署 Statefulset 服务：https://blog.csdn.net/weixin_44729138/article/details/120317584
