python 分布式进程实现

import sys
import os
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.path.pardir))

import Queue
import threading
import multiprocessing
from sfo_common.agent import Agent
from sfo_common.import_common import util, config, time, socket, logger, schedule
from multiprocessing.managers import BaseManager

# 定义一个任务队列
task_queue = Queue.Queue(1000)

# 定义一个类作为server，继承自BaseManager
class SQueueManager(BaseManager):
    pass

# 定义一个类作为client,继承自BaseManager
class CQueueManager(BaseManager):
    pass

def testfunc():
    nowtime = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
    logger.info("server:" + nowtime)

class TaskServer(Agent):
    def __init__(self, pidfile):
        Agent.__init__(self, pidfile)

    def run(self):
        sys.stdout.flush()
        sys.stderr.flush()
        hostname = socket.getfqdn()
        hostip = socket.gethostbyname(hostname)
        logger.info("the task server is running , its hostname is {}, ip is {}".format(hostname, hostip))

        # 将队列注册到网络上
        SQueueManager.register('get_task_queue', callable=lambda: task_queue)
        # 将服务绑定到8102端口上，设置认证key为‘sfo_datahandler’
        manager = SQueueManager(address=('0.0.0.0', 8102), authkey='sfo_datahandler')
        # 启动Queue
        manager.start()
        try:
            # 获得通过网络访问的Queue对象
            task = manager.get_task_queue()
            # 添加任务
            def get_testfunc_schl():
                task.put(testfunc)
            # schdule
            schedule.every(5).seconds.do(get_testfunc_schl)
            schedule.run_all(0)
            # os.wait()
            while True:
                schedule.run_pending()
                time.sleep(0.1)
                # 如果队列满了，说明非提供服务的主机，清空队列
                if task.full():
                    task.clear()
        except Exception as ex:
            logger.exception(ex)
        finally:
            manager.shutdown()


class TaskClient(Agent):
    def __init__(self, pidfile):
        Agent.__init__(self, pidfile)

    def run(self):
        sys.stdout.flush()
        sys.stderr.flush()
        hostname = socket.getfqdn()
        hostip = socket.gethostbyname(hostname)
        logger.info("the task client is running , its hostname is {}, ip is {}".format(hostname, hostip))
        
        # 由于这个queuemanager 只从网络上获取queue，所以注册时只提供名字
        CQueueManager.register('get_task_queue')
        # 连接到服务器，配置的ip为vip，避免单点故障
        manager = CQueueManager(address=(config.task_server, 8102), authkey='sfo_datahandler')
        # 从网络连接
        manager.connect()
        # 获取Queue的对象
        tasks = manager.get_task_queue()
        # 从task队列取任务
        pool = Queue.Queue(config.process_workers)
        while True:
            logger.info("client:" + str(tasks.qsize()))
            if tasks.empty():
                time.sleep(10)
            else:
                if pool.full():
                    time.sleep(10)
                else:
                    work = tasks.get(timeout=5)
                    pool.put(work)
            logger.info("pool:" + str(pool.qsize()))
            if not pool.empty():
                wk = pool.get(1)
                multiprocessing.Process(target=wk).start()


def main():
    util.ensure_dir('/var/log/test/')
    util.ensure_dir(config.sys_agent_pfile)
    util.ensure_dir(config.agent_log_fname)
    ts_pid = '/var/run/test/tsagent.pid'
    tc_pid = '/var/run/test/tcagent.pid'

    tsagent = TaskServer(ts_pid)
    tcagent = TaskClient(tc_pid)
    if len(sys.argv) == 2:
        if 'start' == sys.argv[1]:
            ts = threading.Thread(target=tsagent.start)
            tc = threading.Thread(target=tcagent.start)
            for t in [ts, tc]:
                t.start()
                time.sleep(0.1)
        elif 'stop' == sys.argv[1]:
            tcagent.stop()
            tsagent.stop()
        elif 'restart' == sys.argv[1]:
            ts = threading.Thread(target=tsagent.restart)
            tc = threading.Thread(target=tcagent.restart)
            for t in [ts, tc]:
                t.start()
                time.sleep(0.1)
        else:
            raise IOError.message.format('Unknown command')
    else:
        print("usage: %s" % (sys.argv[1],))
        sys.exit(2)

if __name__ == '__main__':
    main()