zookeeper examples within kazoo

zookeeper样例，包含:

• CRUD
• retry
• watcher
• transaction
• zk异步调用
• zk锁使用

pip requirement

gevent==22.10.2
greenlet==2.0.2
kazoo==2.9.0
selectors2==2.0.2
six==1.16.0
threadpool==1.3.2
zope.event==4.6
zope.interface==5.5.2

docker部署zookeeper

docker run -id --name zk -p 2181:2181 zookeeper:latest

基本操作样例

import sys
import logging
import os
import pdb
from datetime import datetime

import kazoo.exceptions as zkException
from kazoo.client import KazooClient, KazooState, KeeperState
from kazoo.retry import KazooRetry


logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger()

ZK_WORKDIR = "/workspace"
NODE_NAME = ""

zk = KazooClient(hosts="127.0.0.1:2181")
zk.start()


@zk.add_listener
def watch_for_ro(state):
    """
    zk read only mode watcher
    """
    if state == KeeperState.CONNECTED:
        if zk.client_state == KeeperState.CONNECTED_RO:
            logger.info("Read only mode!")
        else:
            logger.info("Read/Write mode!")


@zk.ChildrenWatch(ZK_WORKDIR)
def watch_workdir_children(children):
    """
    workdir children watcher
    """
    logger.info("Workdir children has been changed, children list: {}".format(children))


@zk.DataWatch(ZK_WORKDIR)
def watch_workdir_node(data, stat):
    """
    workdir node watcher
    """
    logger.info("Workdir node has been changed, data: {}".format(data.decode("utf-8")))


def zk_create():
    """
    zk create/ensure_path example
    """
    zk.ensure_path(ZK_WORKDIR)
    # generate current time
    global NODE_NAME
    NODE_NAME = datetime.now().strftime("%Y%m%d__%H_%M_%S")
    zk.create(os.path.join(ZK_WORKDIR, NODE_NAME), bytes(NODE_NAME, "utf-8"))

    zk.set(ZK_WORKDIR, bytes(NODE_NAME, "utf-8"))


def zk_get():
    """
    zk get/exists/get_children example
    """
    full_node_name = os.path.join(ZK_WORKDIR, NODE_NAME)
    if zk.exists(full_node_name):
        logger.info("Node {} exists.".format(NODE_NAME))
    else:
        logger.info("Node {} not exists.".format(NODE_NAME))

    # get node data
    data, stat = zk.get(full_node_name)
    logger.info("Node version: {}, data: {}".format(stat.version, data.decode("utf-8")))

    # list of workspace
    children = zk.get_children(ZK_WORKDIR)
    logger.info("There are {} children with names {}".format(len(children), children))


def zk_set():
    """
    zk set example
    """
    full_node_name = os.path.join(ZK_WORKDIR, NODE_NAME)

    data, stat = zk.get(full_node_name)
    source_data = data.decode("utf-8")

    zk.set(full_node_name, bytes(source_data + "-edited", "utf-8"))

    data, stat = zk.get(full_node_name)
    logger.info("Node version: {}, data: {}".format(stat.version, data.decode("utf-8")))


def zk_del():
    """
    zk delete example
    """
    full_node_name = os.path.join(ZK_WORKDIR, NODE_NAME)
    children = zk.get_children(ZK_WORKDIR)
    logger.info("There are {} children with names {}".format(len(children), children))

    zk.delete(full_node_name, recursive=True)

    children = zk.get_children(ZK_WORKDIR)
    logger.info("There are {} children with names {}".format(len(children), children))


def zk_crud():
    """
    zk crud example
    """
    # zk create/ensure_path
    zk_create()
    # zk get/exists/get_children
    zk_get()
    # zk set
    zk_set()
    # zk del
    zk_del()


def zk_retry():
    """
    zk retry example
    """
    kr = KazooRetry(max_tries=3, ignore_expire=False)
    result = kr(zk.get_children, ZK_WORKDIR)
    logger.info("Retry result: {}".format(result))


def zk_transaction():
    """
    zk transaction example
    """
    transaction = zk.transaction()
    transaction.check(ZK_WORKDIR, version=1)
    node_name = datetime.now().strftime("transaction__%Y%m%d__%H_%M_%S")
    transaction.create(node_name, bytes(node_name, "utf-8"))
    results = transaction.commit()
    logger.info("transaction result: {}".format(results))

    # check each result
    for result in results:
        if isinstance(result, zkException.BadVersionError):
            logger.error("BadVersionError occurred.")
        elif isinstance(result, zkException.RuntimeInconsistency):
            logger.error("RuntimeInconsistency occurred")


def main():
    """
    main logic
    """
    # zk crud example
    zk_crud()
    # zk retry
    # a better retry example maybe distributed lock
    zk_retry()
    # zk watcher example can be seen above
    # zk transaction
    zk_transaction()

    zk.stop()


if __name__ == '__main__':
    main()

zk异步调用样例

import sys
import logging

from kazoo.client import KazooClient
from kazoo.exceptions import ConnectionLossException, NoAuthException
from kazoo.handlers.gevent import SequentialGeventHandler


logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logger = logging.getLogger()

ZK_WORKDIR = "/workspace"

zk = KazooClient(handler=SequentialGeventHandler())

# returns immediately
event = zk.start_async()

# wait for 30 seconds and see if we are connected
event.wait(timeout=30)

if not zk.connected:
    zk.stop()
    raise Exception("Unable to connect zookeeper.")


def async_callback(async_obj):
    """
    async callback
    """
    try:
        children = async_obj.get()
        logger.info("Workspace children list: {}".format(children))
    except (ConnectionLossException, NoAuthException):
        sys.exit(1)


def main():
    """
    main logic
    """
    async_obj = zk.get_children_async(ZK_WORKDIR)
    async_obj.rawlink(async_callback)


if __name__ == "__main__":
    main()

zk锁操作样例

import os
import sys
import threading
import time

from kazoo.client import KazooClient, KazooState


# zookeeper config
ZK_HOST = ZK_DEFAULT_HOST = '127.0.0.1:2181'
LOCK_NAME = 'test_lock'
LOCK_PATH_ROOT = '/'


class ZooKeeperLock():
    """
    zookeeper lock implement
    """

    def __init__(self, hosts, lock_name, timeout=10):
        """
        init zookeeper lock
        """
        if hosts:
            self.hosts = hosts
        else:
            self.hosts = ZK_DEFAULT_HOST
        self.lock_name = lock_name
        self.timeout = timeout

        self._create()

    def _lock_exists(self):
        """
        check whether lock exists
        """
        if self.lock_handle:
            return True
        return False

    def _create(self):
        """
        create zookeeper lock
        """
        # create zookeeper client
        try:
            self.zk = KazooClient(hosts=self.hosts, timeout=self.timeout)
            self.zk.start(timeout=self.timeout)
        except Exception as e:
            self.init_ret = False
            print('Create KazooClient failed! Exception: %s' % str(e))
            return

        # create lock
        try:
            lock_path = os.path.join(LOCK_PATH_ROOT, self.lock_name)
            self.lock_handle = self.zk.Lock(lock_path)
        except Exception as e:
            self.init_ret = False
            print('Create lock failed! Exception: %s' % str(e))
            return

    def destroy(self):
        """
        destroy zookeeper lock
        """
        # check lock status
        if self._lock_exists() and self.lock_handle.is_acquired:
            raise Exception('Destroy lock failed! Please release lock before it has been destroyed.')
        # unlink zookeeper
        if self.zk:
            self.zk.stop()
            self.zk = None

    def acquire(self, blocking=True, timeout=None):
        """
        acquire zookeeper lock
        """
        # check if lock exists
        if not self.lock_handle:
            return None  

        # try acquire lock
        try:
            return self.lock_handle.acquire(blocking=blocking, timeout=timeout)
        except Exception as e:
            print('Acquire lock failed! Exception: %s' % str(e))
            return None

    def release(self):
        """
        release zookeeper lock
        """
        if not self._lock_exists():
            return None
        return self.lock_handle.release()

    def __del__(self):
        """
        del zookeeper lock
        """
        self.release()
        self.destroy()


def create_lock():
    """
    create zookeeper lock
    """
    zk_lock = ZooKeeperLock(ZK_HOST, LOCK_NAME)
    acquire_result = zk_lock.acquire()
    if not acquire_result:
        print('Cannot acquire lock! Result: %s' % acquire_result)
    print('%s acquired lock.' % threading.current_thread().name)
    time.sleep(0.5)
    zk_lock.release()
    print('%s released lock.' % threading.current_thread().name)
    zk_lock.destroy()


def main():
    """
    main logic
    """
    for i in range(0, 10):
        thread = threading.Thread(target=create_lock)
        thread.start()


if __name__ == '__main__':
    main()