python 线程池与进程池简单实现与等待全部线程结束

import time
import random
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import Process, Pool

def worker(n, index):
    print('开始第{}个进程，第{}个线程'.format(n, index))
    t = random.random()
    time.sleep(t)
    print('结束第{}个进程，第{}个线程'.format(n, index))

def main(n):
    max_workers = 20 # 最大线程数
    pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='Thread')
    i = 0
    while True:
        pool.submit(worker, n, i)
        i = i + 1

if __name__ == "__main__":
    pool1 = Pool(2) # 最大进程数2
    for i in range(1,3):
       pool1.apply_async(main, args=(i, ))
    pool1.close()
    pool1.join()

 

等待所有线程结束再进行操作

……

max_workers = 20 # 最大线程数
pool = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='Thread')

task_list = []

while True:
    for _i, _n in enumerate(task_list):
        if _n.done():
            task_list.pop(_i)
    if len(task_list) < int(max_workers * 0.9):
        break  # 判断工作线程数来决定是否添加新的提交新的任务
task_list.append(pool.submit(worker, infos, collection, search_params))

……

 等待所有线程结束再操作并获取线程返回结果

from concurrent.futures import ThreadPoolExecutor, as_completed
……
task_list = []
    if True:
        for n in range(0, max_workers):
            ……
            task_list.append(pool.submit(uptxt, page_start, pasge_end, simhash_index_path_rand))
        while True:
            for lala in as_completed(task_list):
                res = lala.result()  # 获取线程return内容
……    

 其他线程池相关操作：https://blog.csdn.net/xiaoyu_wu/article/details/102820384