Python 多线程、多进程、协成的使用

写在前面将列表分割供多进程、多线程使用

def list_fund(listTemp, n):
    """
    将列表平均分成N块
    :param listTemp:
    :param n:
    :return:
    """
    resules = []
    for i in range(0, len(listTemp), n):
        temp = listTemp[i:i + n]
        resules.append(temp)
    return resules

 

 

 线程池的使用

import time
import threadpool


def sayhello(str):
    print("Hello ", str)
    time.sleep(2)


name_list = ['xiaozi', 'aa', 'bb', 'cc']
start_time = time.time()
pool = threadpool.ThreadPool(4)
requests = threadpool.makeRequests(sayhello, name_list)
[pool.putRequest(req) for req in requests]
pool.wait()
print('%d second' % (time.time() - start_time))

 

多进程的使用 join()

from multiprocessing import Pool

num = 5
p = Pool(num - 1)
for i in range(num):
    p.apply_async(threading_push, args=(part_data_list[i],))

p.close()
p.join()
logger.info("多进程 线程推送成功1 用时:{}".format(round(time.time() - start_time, 4)))


对Pool对象调用join()方法会等待所有子进程执行完毕，调用join()之前必须先调用close()，调用close()之后就不能继续添加新的Process了。

 

进程池的使用

from multiprocessing import Pool

p = Pool(4)  # 创建一个包含5个进程的进程池
print("tuisongshuju:{}".format(pool_data_list))
for i in pool_data_list:
    p.apply_async(func=threading_push, args=(i,))

p.close()  # 等子进程执行完毕后关闭进程池
# time.sleep(2)
# p.terminate()     # 立刻关闭进程池
p.join()

 

协程池

gevent.joinall()等待所有协程结束 timeout设置每个协程超时时间

 

import gevent
import gevent.pool
import gevent.monkey

gevent.monkey.patch_all()  # 分布式冲突
import requests


def download_img(t_event_data):
    """
    保存图片到本地
    :return:
    """
    try:
        index = t_event_data["index"]
        num = t_event_data["num"]
        r = requests.get("https://wwww.baidu.com")
        print("index:{} num:{}".format(index, num))

    except Exception as e:
        print("下载异常:{}".format(e))


def main():
    try:
        num_list = [i for i in range(1, 100)] #待处理的数据
        gevent_data_list = []

        mypool = gevent.pool.Pool(10)  # 定义协程池个数

        for index, num in enumerate(num_list):
            t_event_data = {}
            t_event_data["index"] = index
            t_event_data["num"] = num
            gevent_data_list.append(t_event_data)
            # 协程设置超时时间       
            #gevent_data_list.append(mypool.spawn(t_event_data, t_event_data)) 
        result = mypool.map(download_img, gevent_data_list)
        #result = gevent.joinall(gevent_data_list, timeout=5) # timeout每个协程设置超时时间
        print(result)
    except Exception as e:
        print("异常 {}".format(e))


if __name__ == "__main__":
    main()