Python IO密集型任务、计算密集型任务,以及多线程、多进程
对于IO密集型任务:
- 直接执行用时:10.0333秒
- 多线程执行用时:4.0156秒
- 多进程执行用时:5.0182秒
说明多线程适合IO密集型任务。
对于计算密集型任务
- 直接执行用时:10.0273秒
- 多线程执行用时:13.247秒
- 多进程执行用时:6.8377秒
说明多进程适合计算密集型任务。
#coding=utf-8 import sys import multiprocessing import time import threading # 定义全局变量Queue g_queue = multiprocessing.Queue() def init_queue(): print("init g_queue start") while not g_queue.empty(): g_queue.get() for _index in range(10): g_queue.put(_index) print("init g_queue end") return # 定义一个IO密集型任务:利用time.sleep() def task_io(task_id): print("IOTask[%s] start" % task_id) while not g_queue.empty(): time.sleep(1) try: data = g_queue.get(block=True, timeout=1) print("IOTask[%s] get data: %s" % (task_id, data)) except Exception as excep: print("IOTask[%s] error: %s" % (task_id, str(excep))) print("IOTask[%s] end" % task_id) return g_search_list = list(range(10000)) # 定义一个计算密集型任务:利用一些复杂加减乘除、列表查找等 def task_cpu(task_id): print("CPUTask[%s] start" % task_id) while not g_queue.empty(): count = 0 for i in range(10000): count += pow(3*2, 3*2) if i in g_search_list else 0 try: data = g_queue.get(block=True, timeout=1) print("CPUTask[%s] get data: %s" % (task_id, data)) except Exception as excep: print("CPUTask[%s] error: %s" % (task_id, str(excep))) print("CPUTask[%s] end" % task_id) return task_id if __name__ == '__main__': print("cpu count:", multiprocessing.cpu_count(), "\n") print(u"========== 直接执行IO密集型任务 ==========") init_queue() time_0 = time.time() task_io(0) print(u"结束:", time.time() - time_0, "\n") print("========== 多线程执行IO密集型任务 ==========") init_queue() time_0 = time.time() thread_list = [threading.Thread(target=task_io, args=(i,)) for i in range(10)] for t in thread_list: t.start() for t in thread_list: if t.is_alive(): t.join() print("结束:", time.time() - time_0, "\n") print("========== 多进程执行IO密集型任务 ==========") init_queue() time_0 = time.time() process_list = [multiprocessing.Process(target=task_io, args=(i,)) for i in range(multiprocessing.cpu_count())] for p in process_list: p.start() for p in process_list: if p.is_alive(): p.join() print("结束:", time.time() - time_0, "\n") print("========== 直接执行CPU密集型任务 ==========") init_queue() time_0 = time.time() task_cpu(0) print("结束:", time.time() - time_0, "\n") print("========== 多线程执行CPU密集型任务 ==========") init_queue() time_0 = time.time() thread_list = [threading.Thread(target=task_cpu, args=(i,)) for i in range(10)] for t in thread_list: t.start() for t in thread_list: if t.is_alive(): t.join() print("结束:", time.time() - time_0, "\n") print("========== 多进程执行cpu密集型任务 ==========") init_queue() time_0 = time.time() process_list = [multiprocessing.Process(target=task_cpu, args=(i,)) for i in range(multiprocessing.cpu_count())] for p in process_list: p.start() for p in process_list: if p.is_alive(): p.join() print("结束:", time.time() - time_0, "\n")
参考:https://zhuanlan.zhihu.com/p/24283040