线程池
python没有提供线程池；
1 简单的线程池
#!/usr/bin/env python # -*- coding:utf-8 -*- import Queue import threading class ThreadPool(object): def __init__(self, max_num=20): self.queue = Queue.Queue(max_num) for i in xrange(max_num): self.queue.put(threading.Thread) def get_thread(self): return self.queue.get() def add_thread(self): self.queue.put(threading.Thread) """ pool = ThreadPool(10) def func(arg, p): print arg import time time.sleep(2) p.add_thread() for i in xrange(30): thread = pool.get_thread() t = thread(target=func, args=(i, pool)) t.start() """
#!/usr/bin/env python # _*_ coding:utf-8 _*_ __author__ = 'liujianzuo' import time import queue import threading class MyThreadPool: def __init__(self,max_num): self.queue = queue.Queue(max_num) for i in range(max_num): self.queue.put(threading.Thread) def get_thread(self): return self.queue.get() def add_thread(self): self.queue.put(threading.Thread) def fun(pool ,arg1): time.sleep(1) print(arg1) pool.add_thread() p = MyThreadPool(10) #封装 self.queue 执行__init__的for循环 创建 for i in range(100): thread = p.get_thread() # 获得返回值 threading.Thread 无括号 t = thread(target=fun, args=(p,i)) t.start()
复杂的线程池
#!/usr/bin/env python # _*_ coding:utf-8 _*_ __author__ = 'liujianzuo' import time import queue import threading def outer(func): def inner(*args): ret = func(*args) r1 = callback(ret) return r1 return inner @outer def fun(pool ,arg1): time.sleep(1) # pool.add_thread() return arg1 def callback(args): print(args) r = "" def pool(p,arg): thread = p.get_thread() # 获得返回值 threading.Thread 无括号 t = thread(target=fun, args=(p,arg)) t.start() p.max_num -= 1 if p.max_num == 0: t.join() p.max_num = p.temp # global r # r = p.a(p.max_num) # p.nrange(p.max_num) def multi_thread(max_num,all_num): p = MyThreadPool(max_num) #封装 self.queue 执行__init__的for循环 创建 # p.nrange(p.max_num) # global r # r = p.a(max_num) for i in p.a(all_num): # r.__next__() pool(p,i[0]) class MyThreadPool: def __init__(self,max_num): self.queue = queue.Queue(max_num) self.max_num = max_num self.temp = max_num def a(self,num): temp = -1 while True: temp = temp + 1 if temp >= num: return else: yield temp,self.queue.put(threading.Thread) # def n1(self,stop): # start = 0 # while stop: # yield start # start += 1 # self.queue.put(threading.Thread) # # # def xrange1(self,max_num): # obj = self.n1(max_num) # for x in range(max_num): # obj.__next__() def nrange(self,max_num): for x in self.a(max_num): x def put_thread(self): self.queue.put(threading.Thread) def get_thread(self): return self.queue.get() def add_thread(self): self.queue.put(threading.Thread) multi_thread(6,33)
import threading import time import queue class ThreadPool: def __init__(self, max_num=20): self.queue = queue.Queue(max_num) self.max_pool = max_num self.temp = max_num def mytarget(self): ret = self.func(*self.args) self.callbk(ret) def mypool_handel(self): self.queue.put(threading.Thread) thread = self.queue.get() t = thread(target=self.mytarget) t.start() self.max_pool -= 1 if self.max_pool == 0: t.join() self.max_pool = self.temp def apply(self, myfunc=None, myargs=None, mycallbk=None): self.func = myfunc self.args = myargs self.callbk = mycallbk self.mypool_handel() def f(i): time.sleep(1) return i+100 def f1(i): print(i) p = ThreadPool(4) for i in range(20): p.apply(myfunc=f, myargs=(i,), mycallbk=f1)
上述代码写了一个线程池类，基本实现了线程池的功能，但是有很多缺点，没有实现回掉函数，每次执行任务的时候，任务处理函数每次执行完都需要自动执行对象的add方法，将线程对象添加到队列中去，而且类初始化的时候，一次性将所有的线程类都添加到队列中去了，总之上面的线程池虽然实现简单，但是实际上却有很多问题，下面来看一个真正意义上的线程池。
在写代码之前，我们先来看一下该怎么设计这样一个线程池，上面的线程池，我们的队列中，存的是线程类，我们每处理一个任务都实例化一个线程，然后执行完了之后，该线程就被丢弃了，这样有点不合适。我们这次设计的时候，
- 队列中存的不是线程类，而是任务，我们从队列中拿取的都是任务
- 每次执行任务的时候，不是都要生成一个线程，而是如果以前生成的线程有空闲的话，就用以前的线程
- 支持回掉机制，支持close，terminate
下面来一下代码是怎么实现的
#!/usr/bin/env python
# _*_ coding:utf-8 _*_
__author__ = 'liujianzuo'
"""
custom ThreadPool
How to use:
pool = ThreadPool(1)
def callback(status, result):
# status, execute action status
# result, execute action return value
pass
def action(i):
pass
for i in range(20):
if pool.stop:
pool.terminal()
break
ret = pool.run(action, (i,), callback)
print 'end'
"""
import queue
import threading
import contextlib
StopEvent = object() # 由于我们是将创建的任务方队列让线程去取，而我们下面的定义的任务是元祖，只要我们放一个停止信号不是元祖即可，这里我们随意定义。
class ThreadPool:
def __init__(self,max_num):
self.q = queue.Queue(max_num)
self.max_num = max_num
self.cancel = False
self.generate_list = [] # 实际线程处理任务的列表，我们的任务队列有10个，但是我不一定有10个线程处理。
self.free_list = [] # 处理完任务的线程我们放到这个列表，空闲线程列表
def run(self, func, args, callback=None):
"""
线程池执行一个任务
:param func: 任务函数
:param args: 任务函数所需参数
:param callback: 任务执行失败或成功后执行的回调函数，回调函数有两个参数1、任务函数执行状态；2、任务函数返回值（默认为None，即：不执行回调函数）
:return: 如果线程池已经终止，则返回True否则None
"""
if len(self.free_list) == 0 and len(self.generate_list) < self.max_num:
self.generate_thread()
w = (func, args, callback,) # 任务格式是元祖
self.q.put(w)
def generate_thread(self):
"""
创建一个线程
"""
t = threading.Thread(target=self.call)
t.start()
def call(self):
"""
循环去获取任务函数并执行任务函数
"""
current_thread = threading.currentThread # 获取当前的线程
self.generate_list.append(current_thread)
event = self.q.get() #取一个任务
while event != StopEvent:# 判断任务是不是停止信号 # 这里用while，如果一个线程处理的块，就不用启动太多线程，线程列表会很少
func, arguments, callback = event
try: #执行任务万一出错
result = func(*arguments)
status = True
except Exception as e:
status = False
result = e
if callback is not None: # 没有传callback的话也可以
try:
callback(status, result) # 回调函数处理
except Exception as e:
pass
"""#如下三句可以用ｃｏｎｔｅｘｌｉｂ的上下文管理实现
self.free_list.append(current_thread) # 处理完后将线程交给空闲列表
event = self.q.get() # 再取一个任务
self.free_list.remove(current_thread) #删除空闲列表 未删除前，有个启动判断
"""
with self.worker_state(self.free_list, current_thread):
event = self.q.get()
else:
self.generate_list.remove(current_thread) #如果遇到停止信号，当前线程就移除可用列表
def close(self): # 启动多线程后，如果后面有调用close方法，就改变状态
"""
执行完所有的任务后，所有线程停止
"""
self.cancel = True
full_size = len(self.generate_list)
while full_size:
self.q.put(StopEvent)
full_size -= 1
def terminate(self):
"""
无论是否还有任务，终止线程
"""
self.terminal = True
while self.generate_list: # 已经有的任务让其处理完，然后放置一个停止信号
self.q.put(StopEvent)
self.q.empty() #然后清空队列
@contextlib.contextmanager
def worker_state(self, state_list, worker_thread): #上下文管理，做一件事后，
"""
用于记录线程中正在等待的线程数
"""
state_list.append(worker_thread)
try:
yield
finally:
state_list.remove(worker_thread)
"""
# How to use
pool = ThreadPool(5)
def callback(status, result):
# status, execute action status
# result, execute action return value
pass
def action(i):
time.sleep(1)
print(i)
for i in range(30):
ret = pool.run(action, (i,), callback)
# pool.close()
# pool.terminate()
"""
#!/usr/bin/env python # -*- coding:utf-8 -*- import queue import threading import contextlib import time StopEvent = object() class ThreadPool(object): def __init__(self, max_num): self.q = queue.Queue() self.max_num = max_num self.terminal = False self.generate_list = [] self.free_list = [] def run(self, func, args, callback=None): """ 线程池执行一个任务 :param func: 任务函数 :param args: 任务函数所需参数 :param callback: 任务执行失败或成功后执行的回调函数，回调函数有两个参数1、任务函数执行状态；2、任务函数返回值（默认为None，即：不执行回调函数） :return: 如果线程池已经终止，则返回True否则None """ if len(self.free_list) == 0 and len(self.generate_list) < self.max_num: self.generate_thread() w = (func, args, callback,) self.q.put(w) def generate_thread(self): """ 创建一个线程 """ t = threading.Thread(target=self.call) t.start() def call(self): """ 循环去获取任务函数并执行任务函数 """ current_thread = threading.currentThread self.generate_list.append(current_thread) event = self.q.get() while event != StopEvent: func, arguments, callback = event try: result = func(*arguments) success = True except Exception as e: success = False result = None if callback is not None: try: callback(success, result) except Exception as e: pass with self.worker_state(self.free_list, current_thread): if self.terminal: event = StopEvent else: event = self.q.get() else: self.generate_list.remove(current_thread) def close(self): """ 执行完所有的任务后，所有线程停止 """ full_size = len(self.generate_list) while full_size: self.q.put(StopEvent) full_size -= 1 def terminate(self): """ 无论是否还有任务，终止线程 """ self.terminal = True while self.generate_list: self.q.put(StopEvent) self.q.empty() @contextlib.contextmanager def worker_state(self, state_list, worker_thread): """ 用于记录线程中正在等待的线程数 """ state_list.append(worker_thread) try: yield finally: state_list.remove(worker_thread) """ # How to use pool = ThreadPool(5) def callback(status, result): # status, execute action status # result, execute action return value pass def action(i): time.sleep(1) print(i) for i in range(30): ret = pool.run(action, (i,), callback) # pool.close() # pool.terminate() """
#!/usr/bin/env python # -*- coding:utf-8 -*- import queue import threading import contextlib import time StopEvent = object() class ThreadPool(object): def __init__(self, max_num, max_task_num = None): if max_task_num: self.q = queue.Queue(max_task_num) else: self.q = queue.Queue() self.max_num = max_num self.cancel = False self.terminal = False self.generate_list = [] self.free_list = [] def run(self, func, args, callback=None): """ 线程池执行一个任务 :param func: 任务函数 :param args: 任务函数所需参数 :param callback: 任务执行失败或成功后执行的回调函数，回调函数有两个参数1、任务函数执行状态；2、任务函数返回值（默认为None，即：不执行回调函数） :return: 如果线程池已经终止，则返回True否则None """ if self.cancel: return if len(self.free_list) == 0 and len(self.generate_list) < self.max_num: self.generate_thread() w = (func, args, callback,) self.q.put(w) def generate_thread(self): """ 创建一个线程 """ t = threading.Thread(target=self.call) t.start() def call(self): """ 循环去获取任务函数并执行任务函数 """ current_thread = threading.currentThread self.generate_list.append(current_thread) event = self.q.get() while event != StopEvent: func, arguments, callback = event try: result = func(*arguments) success = True except Exception as e: success = False result = None if callback is not None: try: callback(success, result) except Exception as e: pass with self.worker_state(self.free_list, current_thread): if self.terminal: event = StopEvent else: event = self.q.get() else: self.generate_list.remove(current_thread) def close(self): """ 执行完所有的任务后，所有线程停止 """ self.cancel = True full_size = len(self.generate_list) while full_size: self.q.put(StopEvent) full_size -= 1 def terminate(self): """ 无论是否还有任务，终止线程 """ self.terminal = True while self.generate_list: self.q.put(StopEvent) self.q.empty() @contextlib.contextmanager def worker_state(self, state_list, worker_thread): """ 用于记录线程中正在等待的线程数 """ state_list.append(worker_thread) try: yield finally: state_list.remove(worker_thread) # How to use pool = ThreadPool(5) def callback(status, result): # status, execute action status # result, execute action return value pass def action(i): print(i) for i in range(30): ret = pool.run(action, (i,), callback) time.sleep(5) print(len(pool.generate_list), len(pool.free_list)) print(len(pool.generate_list), len(pool.free_list)) # pool.close() # pool.terminate()
