Python多进程、多线程与协程
多进程: 适合处理CPU密集型任务,利用多核处理器并行计算。
多线程: 适合IO密集型任务,因为线程可以在等待IO完成时释放GIL(全局解释器锁),允许其他线程运行。
协程: 适合处理大量并发的IO密集型任务,相比于传统的线程切换,协程切换的开销更小。
多进程
不使用多进程,统计耗时
import time
# CPU密集型任务,例如计算斐波那契数列
def fibonacci(n):
if n <= 1:
return n
else:
return fibonacci(n - 1) + fibonacci(n - 2)
if __name__ == '__main__':
numbers = [30, 31, 32, 33]
# 统计耗时
start_time = time.time()
results_single = [fibonacci(num) for num in numbers]
end_time = time.time()
print("Results:", results_single)
print("Time Cost:", end_time - start_time, "seconds")
# 耗时约2.6s
使用multiprocessing库
import multiprocessing
import time
# CPU密集型任务,例如计算斐波那契数列
def fibonacci(n):
if n <= 1:
return n
else:
return fibonacci(n - 1) + fibonacci(n - 2)
if __name__ == '__main__':
# 创建一个进程池,这里使用默认的CPU核心数
pool = multiprocessing.Pool()
numbers = [30, 31, 32, 33]
start_time = time.time()
# 使用进程池并行计算每个数的斐波那契数
results = pool.map(fibonacci, numbers)
end_time = time.time()
# 关闭进程池
pool.close()
pool.join()
# 打印结果和耗时
print("Results:", results)
print("Time Cost with multiprocessing:", end_time - start_time, "seconds")
# 耗时约1.3s
多线程
不使用多线程,统计耗时
import requests
import time
# 免费可用于测试的API
# https://jsonplaceholder.typicode.com/
# Free fake and reliable API for testing and prototyping.
urls = [
'https://jsonplaceholder.typicode.com/posts/1',
'https://jsonplaceholder.typicode.com/posts/2',
'https://jsonplaceholder.typicode.com/posts/3',
'https://jsonplaceholder.typicode.com/posts/4',
'https://jsonplaceholder.typicode.com/posts/5'
]
# IO密集型任务,例如网络请求
def fetch_data(url):
response = requests.get(url)
return response.json()
if __name__ == '__main__':
start_time = time.time()
results = [fetch_data(url) for url in urls]
end_time = time.time()
print("Time Cost:", end_time - start_time, "seconds")
# 耗时约3.0s
使用threading模块创建线程
import requests
import threading
import time
urls = [
'https://jsonplaceholder.typicode.com/posts/1',
'https://jsonplaceholder.typicode.com/posts/2',
'https://jsonplaceholder.typicode.com/posts/3',
'https://jsonplaceholder.typicode.com/posts/4',
'https://jsonplaceholder.typicode.com/posts/5'
]
def fetch_data(url):
response = requests.get(url)
return response.json()
if __name__ == '__main__':
start_time = time.time()
# 为每个URL创建一个线程
threads = [threading.Thread(target=fetch_data, args=(url,)) for url in urls]
# 启动所有线程
for thread in threads:
thread.start()
# 确保所有线程结束
for thread in threads:
thread.join()
end_time = time.time()
print("Time taken with threading:", end_time - start_time, "seconds")
# 耗时约0.68s
使用线程池方式实现
import requests
import concurrent.futures
import time
urls = [
'https://jsonplaceholder.typicode.com/posts/1',
'https://jsonplaceholder.typicode.com/posts/2',
'https://jsonplaceholder.typicode.com/posts/3',
'https://jsonplaceholder.typicode.com/posts/4',
'https://jsonplaceholder.typicode.com/posts/5'
]
def fetch_data(url):
response = requests.get(url)
return response.json()
if __name__ == '__main__':
start_time = time.time()
# Create a thread pool with a maximum of 5 threads (one for each URL)
with concurrent.futures.ThreadPoolExecutor(max_workers=len(urls)) as executor:
# Submit each URL fetching task to the thread pool
futures = [executor.submit(fetch_data, url) for url in urls]
# Wait for all futures (tasks) to complete
results = [future.result() for future in concurrent.futures.as_completed(futures)]
end_time = time.time()
print("Results with thread pool:", results)
print("Time taken with thread pool:", end_time - start_time, "seconds")
# 耗时约0.84s
协程
使用协程方式实现
import asyncio
import aiohttp
import time
urls = [
'https://jsonplaceholder.typicode.com/posts/1',
'https://jsonplaceholder.typicode.com/posts/2',
'https://jsonplaceholder.typicode.com/posts/3',
'https://jsonplaceholder.typicode.com/posts/4',
'https://jsonplaceholder.typicode.com/posts/5'
]
# 协程方式获取数据
async def fetch_data(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
return await response.json()
async def main():
start_time = time.time()
# Gather all coroutine tasks for fetching data
tasks = [fetch_data(url) for url in urls]
results = await asyncio.gather(*tasks)
end_time = time.time()
print("Results with coroutines:", results)
print("Time taken with coroutines:", end_time - start_time, "seconds")
if __name__ == '__main__':
asyncio.run(main())
# 耗时约0.70s