Python3 协程并发对比串行下载文件
通过 asyncio 和 aiohttp 协程下载文件控制在3秒以内 串行要20秒
import time import aiohttp import asyncio import requests def down_img(url, i): r = requests.get(url) save_path = f"/tmp/123/{i}.jpg" with open(save_path, "wb") as pdf: for chunk in r.iter_content(chunk_size=2048): if chunk: pdf.write(chunk) async def job(session, index, url): # 声明为异步函数 file = await session.get(url, verify_ssl=False) # 触发到await就切换,等待get到数据 filecode = await file.read() # 读取内容 with open(f'/tmp/123/{index}.jpg', 'wb') as f: # 写入到指定目录下的文件中 f.write(filecode) return str(url) async def main(loop, URL): async with aiohttp.ClientSession() as session: # 建立会话session tasks = [] # 建立所有任务 for index, url in enumerate(URL): tasks.append(loop.create_task(job(session, index, url))) finished, unfinished = await asyncio.wait(tasks) # 触发await,等待任务完成 # all_results = [r.result() for r in finished] # 获取所有结果 # print("all_results:", all_results) URL = [ "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg", ] t1 = time.time() # 协程方式3秒以内 loop = asyncio.get_event_loop() loop.run_until_complete(main(loop, URL)) loop.close() # 串行20秒以上 # for index, url in enumerate(URL): # down_img(url, index) t2 = time.time() print("time:", t2 - t1)