python 收录集中实现线程池的方法

概念：

什么是线程池？

      诸如web服务器、数据库服务器、文件服务器和邮件服务器等许多服务器应用都面向处理来自某些远程来源的大量短小的任务。构建服务器应用程序的一个过于简单的模型是：每当一个请求到达就创建一个新的服务对象，然后在新的服务对象中为请求服务。但当有大量请求并发访问时，服务器不断的创建和销毁对象的开销很大。所以提高服务器效率的一个手段就是尽可能减少创建和销毁对象的次数，特别是一些很耗资源的对象创建和销毁，这样就引入了“池”的概念，“池”的概念使得人们可以定制一定量的资源，然后对这些资源进行复用，而不是频繁的创建和销毁。

      线程池是预先创建线程的一种技术。线程池在还没有任务到来之前，创建一定数量的线程，放入空闲队列中。这些线程都是处于睡眠状态，即均为启动，不消耗CPU，而只是占用较小的内存空间。当请求到来之后，缓冲池给这次请求分配一个空闲线程，把请求传入此线程中运行，进行处理。当预先创建的线程都处于运行状态，即预制线程不够，线程池可以自由创建一定数量的新线程，用于处理更多的请求。当系统比较闲的时候，也可以通过移除一部分一直处于停用状态的线程。

 

1，简易方法，利用队列实现

（1）

 

from threading import Thread

import time, os, queue

class Threading_Pool(object):

    def __init__(self, MAX_NUM = 20):

        self.queue = queue.Queue(MAX_NUM)
        # self.queue.join()
        for i in range(MAX_NUM):
            self.queue.put(Thread)

    def get_thread(self):
        return self.queue.get()

    def put_thread(self):
        self.queue.put(Thread)

    def join(self):

        self.queue.join()


def test(tp):
    time.sleep(1)
    print('haha')

    tp.put_thread()
    # print(tp.queue.full())获取队列是否满了

if __name__ == '__main__':
    tp = Threading_Pool(5)
    # tp.join()
    for i in range(20):
        p = tp.get_thread()
        t1 = p(target = test, args = (tp, ))
        t1.start()

打印结果会1秒出5个

 

　　

 

（2）

# -*- coding:utf-8 -*-  
  
import Queue  
import threading  
import time  
  
class WorkManager(object):  
    def __init__(self, work_num=1000,thread_num=2):  
        self.work_queue = Queue.Queue()  
        self.threads = []  
        self.__init_work_queue(work_num)  
        self.__init_thread_pool(thread_num)  
  
    """ 
        初始化线程 
    """  
    def __init_thread_pool(self,thread_num):  
        for i in range(thread_num):  
            self.threads.append(Work(self.work_queue))  
  
    """ 
        初始化工作队列 
    """  
    def __init_work_queue(self, jobs_num):  
        for i in range(jobs_num):  
            self.add_job(do_job, i)  
  
    """ 
        添加一项工作入队 
    """  
    def add_job(self, func, *args):  
        self.work_queue.put((func, list(args)))#任务入队，Queue内部实现了同步机制  
  
    """ 
        等待所有线程运行完毕 
    """     
    def wait_allcomplete(self):  
        for item in self.threads:  
            if item.isAlive():item.join()  
  
class Work(threading.Thread):  
    def __init__(self, work_queue):  
        threading.Thread.__init__(self)  
        self.work_queue = work_queue  
        self.start()  
  
    def run(self):  
        #死循环，从而让创建的线程在一定条件下关闭退出  
        while True:  
            try:  
                do, args = self.work_queue.get(block=False)#任务异步出队，Queue内部实现了同步机制  
                do(args)  
                self.work_queue.task_done()#通知系统任务完成  
            except:  
                break  
  
#具体要做的任务  
def do_job(args):  
    time.sleep(0.1)#模拟处理时间  
    print threading.current_thread(), list(args)  
  
if __name__ == '__main__':  
    start = time.time()  
    work_manager =  WorkManager(10000, 10)#或者work_manager =  WorkManager(10000, 20)  
    work_manager.wait_allcomplete()  
    end = time.time()  
    print "cost all time: %s" % (end-start)  


  #2次开启不同的线程数运行结果如下：
 #work_manager =  WorkManager(10000, 10)  
 cost all time: 100.641790867(单位：秒)  
 #work_manager =  WorkManager(10000, 20)  
 cost all time：50.5233478546(单位：秒)  

 

2，高效方法（1）

#-*-encoding:utf-8-*-
'''
Created on 2012-3-9
@summary: 线程池
@contact: mailto:zhanglixinseu@gmail.com
@author: zhanglixin
'''
import sys
import threading
import Queue
import traceback

# 定义一些Exception，用于自定义异常处理

class NoResultsPending(Exception):
    """All works requests have been processed"""
    pass

class NoWorkersAvailable(Exception):
    """No worket threads available to process remaining requests."""
    pass

def _handle_thread_exception(request, exc_info):
    """默认的异常处理函数，只是简单的打印"""
    traceback.print_exception(*exc_info)

#classes 

class WorkerThread(threading.Thread):
    """后台线程，真正的工作线程，从请求队列(requestQueue)中获取work，
    并将执行后的结果添加到结果队列(resultQueue)"""
    def __init__(self,requestQueue,resultQueue,poll_timeout=5,**kwds):
        threading.Thread.__init__(self,**kwds)
        '''设置为守护进行'''
        self.setDaemon(True)
        self._requestQueue = requestQueue
        self._resultQueue = resultQueue
        self._poll_timeout = poll_timeout
        '''设置一个flag信号，用来表示该线程是否还被dismiss,默认为false'''
        self._dismissed = threading.Event()
        self.start()
        
    def run(self):
        '''每个线程尽可能多的执行work，所以采用loop，
        只要线程可用，并且requestQueue有work未完成，则一直loop'''
        while True:
            if self._dismissed.is_set():
                break
            try:
                '''
                Queue.Queue队列设置了线程同步策略，并且可以设置timeout。
                一直block，直到requestQueue有值，或者超时
                '''
                request = self._requestQueue.get(True,self._poll_timeout)
            except Queue.Empty:
                continue
            else:
                '''之所以在这里再次判断dimissed，是因为之前的timeout时间里，很有可能，该线程被dismiss掉了'''
                if self._dismissed.is_set():
                    self._requestQueue.put(request)
                    break
                try:
                    '''执行callable，讲请求和结果以tuple的方式放入requestQueue'''
                    result = request.callable(*request.args,**request.kwds)
                    print self.getName()
                    self._resultQueue.put((request,result))
                except:
                    '''异常处理'''
                    request.exception = True
                    self._resultQueue.put((request,sys.exc_info()))
    
    def dismiss(self):
        '''设置一个标志，表示完成当前work之后，退出'''
        self._dismissed.set()


class WorkRequest:
    '''
    @param callable_:，可定制的，执行work的函数
    @param args: 列表参数
    @param kwds: 字典参数
    @param requestID: id
    @param callback: 可定制的，处理resultQueue队列元素的函数
    @param exc_callback:可定制的，处理异常的函数 
    '''
    def __init__(self,callable_,args=None,kwds=None,requestID=None,
                 callback=None,exc_callback=_handle_thread_exception):
        if requestID == None:
            self.requestID = id(self)
        else:
            try:
                self.requestID = hash(requestID)
            except TypeError:
                raise TypeError("requestId must be hashable")    
        self.exception = False
        self.callback = callback
        self.exc_callback = exc_callback
        self.callable = callable_
        self.args = args or []
        self.kwds = kwds or {}
        
    def __str__(self):
        return "WorkRequest id=%s args=%r kwargs=%r exception=%s" % \
            (self.requestID,self.args,self.kwds,self.exception)
            
class ThreadPool:
    '''
    @param num_workers:初始化的线程数量
    @param q_size,resq_size: requestQueue和result队列的初始大小
    @param poll_timeout: 设置工作线程WorkerThread的timeout，也就是等待requestQueue的timeout
    '''
    def __init__(self,num_workers,q_size=0,resq_size=0,poll_timeout=5):
        self._requestQueue = Queue.Queue(q_size)
        self._resultQueue = Queue.Queue(resq_size)
        self.workers = []
        self.dismissedWorkers = []
        self.workRequests = {} #设置个字典，方便使用
        self.createWorkers(num_workers,poll_timeout)

    def createWorkers(self,num_workers,poll_timeout=5):
        '''创建num_workers个WorkThread,默认timeout为5'''
        for i in range(num_workers):
            self.workers.append(WorkerThread(self._requestQueue,self._resultQueue,poll_timeout=poll_timeout))                           
    
    def dismissWorkers(self,num_workers,do_join=False):
        '''停用num_workers数量的线程，并加入dismiss_list'''
        dismiss_list = []
        for i in range(min(num_workers,len(self.workers))):
            worker = self.workers.pop()
            worker.dismiss()
            dismiss_list.append(worker)
        if do_join :
            for worker in dismiss_list:
                worker.join()
        else:
            self.dismissedWorkers.extend(dismiss_list)
    
    def joinAllDismissedWorkers(self):
        '''join 所有停用的thread'''
        #print len(self.dismissedWorkers)
        for worker in self.dismissedWorkers:
            worker.join()
        self.dismissedWorkers = []
    
    def putRequest(self,request ,block=True,timeout=None):
        assert isinstance(request,WorkRequest)
        assert not getattr(request,'exception',None)
        '''当queue满了，也就是容量达到了前面设定的q_size,它将一直阻塞，直到有空余位置，或是timeout'''
        self._requestQueue.put(request, block, timeout)
        self.workRequests[request.requestID] = request
        
    def poll(self,block = False):
        while True:
            if not self.workRequests:
                raise NoResultsPending
            elif block and not self.workers:
                raise NoWorkersAvailable
            try:
                '''默认只要resultQueue有值，则取出，否则一直block'''
                request , result = self._resultQueue.get(block=block)
                if request.exception and request.exc_callback:
                    request.exc_callback(request,result)
                if request.callback and not (request.exception and request.exc_callback):
                    request.callback(request,result)
                del self.workRequests[request.requestID]
            except Queue.Empty:
                break
    
    def wait(self):
        while True:
            try:
                self.poll(True)
            except NoResultsPending:
                break
    
    def workersize(self):
        return len(self.workers)
    
    def stop(self):
        '''join 所有的thread,确保所有的线程都执行完毕'''
        self.dismissWorkers(self.workersize(),True)
        self.joinAllDismissedWorkers()

测试代码：

#Test a demo

if __name__=='__main__':
    import random
    import time
    import datetime
    def do_work(data):
        time.sleep(random.randint(1,3))
        res = str(datetime.datetime.now()) + "" +str(data)
        return res
    
    def print_result(request,result):
        print "---Result from request %s : %r" % (request.requestID,result)
    
    main = ThreadPool(3)
    for i in range(40):
        req = WorkRequest(do_work,args=[i],kwds={},callback=print_result)
        main.putRequest(req)
        print "work request #%s added." % req.requestID
    
    print '-'*20, main.workersize(),'-'*20
    
    counter = 0
    while True:
        try:
            time.sleep(0.5)
            main.poll()
            if(counter==5):
                print "Add 3 more workers threads"
                main.createWorkers(3)
                print '-'*20, main.workersize(),'-'*20
            if(counter==10):
                print "dismiss 2 workers threads"
                main.dismissWorkers(2)
                print '-'*20, main.workersize(),'-'*20
            counter+=1
        except NoResultsPending:
            print "no pending results"
            break
    
    main.stop()
    print "Stop"

高效版（2）（老外所作）

import threading  
  
class WorkerTask(object):  
    """A task to be performed by the ThreadPool."""  
  
    def __init__(self, function, args=(), kwargs={}):  
        self.function = function  
        self.args = args  
        self.kwargs = kwargs  
  
    def __call__(self):  
        self.function(*self.args, **self.kwargs)  
  
  
class WorkerThread(threading.Thread):  
    """A thread managed by a thread pool."""  
  
    def __init__(self, pool):  
        threading.Thread.__init__(self)  
        self.setDaemon(True)  
        self.pool = pool  
        self.busy = False  
        self._started = False  
        self._event = None  
  
    def work(self):  
        if self._started is True:  
            if self._event is not None and not self._event.isSet():  
                self._event.set()  
        else:  
            self._started = True  
            self.start()  
  
    def run(self):  
        while True:  
            self.busy = True  
            while len(self.pool._tasks) > 0:  
                try:  
                    task = self.pool._tasks.pop()  
                    task()  
                except IndexError:  
                    # Just in case another thread grabbed the task 1st.  
                    pass  
  
            # Sleep until needed again  
            self.busy = False  
            if self._event is None:  
                self._event = threading.Event()  
            else:  
                self._event.clear()  
            self._event.wait()  
  
class ThreadPool(object):  
    """Executes queued tasks in the background."""  
  
    def __init__(self, max_pool_size=10):  
        self.max_pool_size = max_pool_size  
        self._threads = []  
        self._tasks = []   
  
    def _addTask(self, task):  
        self._tasks.append(task)  
  
        worker_thread = None  
        for thread in self._threads:  
            if thread.busy is False:  
                worker_thread = thread  
                break  
  
        if worker_thread is None and len(self._threads) <= self.max_pool_size:  
            worker_thread = WorkerThread(self)  
            self._threads.append(worker_thread)  
  
        if worker_thread is not None:  
            worker_thread.work()  
  
    def addTask(self, function, args=(), kwargs={}):  
        self._addTask(WorkerTask(function, args, kwargs))  
  
class GlobalThreadPool(object):  
    """ThreadPool Singleton class."""  
  
    _instance = None  
  
    def __init__(self):  
        """Create singleton instance """  
  
        if GlobalThreadPool._instance is None:  
            # Create and remember instance  
            GlobalThreadPool._instance = ThreadPool()  
  
    def __getattr__(self, attr):  
        """ Delegate get access to implementation """  
        return getattr(self._instance, attr)  
  
    def __setattr__(self, attr, val):  
        """ Delegate set access to implementation """  
        return setattr(self._instance, attr, val)