python多线程与多进程异步事件框架
多线程简单实现
#!/usr/bin/env python # -*- coding: UTF-8 -*- import logging import queue import threading from concurrent.futures import ThreadPoolExecutor # 任务：事件 def func_a(a, b): return a + b def func_b(a, b): return a * b def func_c(a, b, c): return a * b - c # 回调函数 def handle_result1(result): print(type(result), result) def handle_result2(result): print(type(result), result) def handle_result3(result): print(type(result), result) class EventEngine(object): # 初始化事件事件驱动引擎 def __init__(self): # 保存事件列表:异步任务队列 self.__eventQueue = queue.Queue() # 引擎开关 self.__active = False # 事件处理字典{'event1': [handler1,handler2] , 'event2':[handler3, ...,handler4]} self.__handlers = {} # 事件引擎主进程 self.__Thread = threading.Thread(target=self.task_queue_consumer) # 事件处理线程池 self.__thread_pool = ThreadPoolExecutor(max_workers=5) # 线程处理存储 self.__thread_Pool = [] #注册事件 def register(self,event, callback, *args, **kwargs): Event = { 'function': event, 'callback': callback, 'args': args, 'kwargs': kwargs } self.__handlers[event] = Event #注销事件 def unregister(self,event): if(self.__handlers[event]): del self.__handlers[event] #提交事件 def sendevent(self,event): if ( event in self.__handlers.keys()): self.__eventQueue.put(self.__handlers[event]) # 开启事件引擎 def start(self): self.__active = True self.__Thread.start() # 暂停事件引擎 def stop(self): self.__active = False # 暂停后开始 def restart(self): self.__active = True # 关闭事件引擎 def close(self): pass # 开启事件循环 def task_queue_consumer(self): """ 异步任务队列 """ while(1): while self.__active: if (self.__eventQueue.empty() == False): try: task = self.__eventQueue.get() function = task.get('function') callback = task.get('callback') args = task.get('args') kwargs = task.get('kwargs') try: if callback: thread = self.__thread_pool.submit(callback,function(*args, **kwargs)) self.__thread_Pool.append(thread) # callback(function(*args, **kwargs)) except Exception as ex: if callback: callback(ex) finally: self.__eventQueue.task_done() except Exception as ex: logging.warning(ex) if __name__ == '__main__': import time #初始化多线程异步框架 Engine = EventEngine() #启动 Engine.start() #注册回调函数 Engine.register(func_a, handle_result1, 1, 2) Engine.register(func_b, handle_result2, 1, 2) Engine.register(func_c, handle_result3, 1, 2, 3) #提交事件 Engine.sendevent(func_a) Engine.sendevent(func_b) Engine.sendevent(func_c) time.sleep(2) Engine.stop() Engine.restart() Engine.sendevent(func_b) Engine.sendevent(func_c) # for i in range(100): # Engine.sendevent(func_a)
多进程实现
from multiprocessing import Process, Queue class EventEngine(object): # 初始化事件事件驱动引擎 def __init__(self): #保存事件列表 self.__eventQueue = Queue() #引擎开关 self.__active = False #事件处理字典{'event1': [handler1,handler2] , 'event2':[handler3, ...,handler4]} self.__handlers = {} #保存事件处理进程池 self.__processPool = [] #事件引擎主进程 self.__mainProcess = Process(target=self.__run) #执行事件循环 def __run(self): while self.__active: #事件队列非空 if not self.__eventQueue.empty(): #获取队列中的事件 超时1秒 event = self.__eventQueue.get(block=True ,timeout=1) #执行事件 self.__process(event) else: # print('无任何事件') pass #执行事件 def __process(self, event): if event.type in self.__handlers: for handler in self.__handlers[event.type]: #开一个进程去异步处理 p = Process(target=handler, args=(event, )) #保存到进程池 self.__processPool.append(p) p.start() #开启事件引擎 def start(self): self.__active = True self.__mainProcess.start() #暂停事件引擎 def stop(self): """停止""" # 将事件管理器设为停止 self.__active = False # 等待事件处理进程退出 for p in self.__processPool: p.join() self.__mainProcess.join() #终止事件引擎 def terminate(self): self.__active = False #终止所有事件处理进程 for p in self.__processPool: p.terminate() self.__mainProcess.join() #注册事件 def register(self, type, handler): """注册事件处理函数监听""" # 尝试获取该事件类型对应的处理函数列表，若无则创建 try: handlerList = self.__handlers[type] except KeyError: handlerList = [] self.__handlers[type] = handlerList # 若要注册的处理器不在该事件的处理器列表中，则注册该事件 if handler not in handlerList: handlerList.append(handler) def unregister(self, type, handler): """注销事件处理函数监听""" # 尝试获取该事件类型对应的处理函数列表，若无则忽略该次注销请求 try: handlerList = self.__handlers[type] # 如果该函数存在于列表中，则移除 if handler in handlerList: handlerList.remove(handler) # 如果函数列表为空，则从引擎中移除该事件类型 if not handlerList: del self.__handlers[type] except KeyError: pass def sendEvent(self, event): #发送事件 像队列里存入事件 self.__eventQueue.put(event) class Event(object): #事件对象 def __init__(self, type =None): self.type = type self.dict = {} #测试 if __name__ == '__main__': import time EVENT_ARTICAL = "Event_Artical" # 事件源 公众号 class PublicAccounts: def __init__(self, eventManager): self.__eventManager = eventManager def writeNewArtical(self): # 事件对象，写了新文章 event = Event(EVENT_ARTICAL) event.dict["artical"] = u'如何写出更优雅的代码\n' # 发送事件 self.__eventManager.sendEvent(event) print(u'公众号发送新文章\n') # 监听器 订阅者 class ListenerTypeOne: def __init__(self, username): self.__username = username # 监听器的处理函数 读文章 def ReadArtical(self, event): print(u'%s 收到新文章' % self.__username) print(u'%s 正在阅读新文章内容：%s' % (self.__username, event.dict["artical"])) class ListenerTypeTwo: def __init__(self, username): self.__username = username # 监听器的处理函数 读文章 def ReadArtical(self, event): print(u'%s 收到新文章 睡3秒再看' % self.__username) time.sleep(3) print(u'%s 正在阅读新文章内容：%s' % (self.__username, event.dict["artical"])) def test(): listner1 = ListenerTypeOne("thinkroom") # 订阅者1 listner2 = ListenerTypeTwo("steve") # 订阅者2 ee = EventEngine() # 绑定事件和监听器响应函数(新文章) ee.register(EVENT_ARTICAL, listner1.ReadArtical) ee.register(EVENT_ARTICAL, listner2.ReadArtical) for i in range(0, 20): listner3 = ListenerTypeOne("Jimmy") # 订阅者X ee.register(EVENT_ARTICAL, listner3.ReadArtical) ee.start() #发送事件 publicAcc = PublicAccounts(ee) publicAcc.writeNewArtical() test()
多进程程序来源：http://blog.sina.com.cn/s/blog_13bb711fd0102x5nd.html
