Python3 协程并发对比串行下载文件

通过 asyncio 和 aiohttp 协程下载文件控制在3秒以内 串行要20秒

 

import time
import aiohttp
import asyncio
import requests


def down_img(url, i):
    r = requests.get(url)
    save_path = f"/tmp/123/{i}.jpg"
    with open(save_path, "wb") as pdf:
        for chunk in r.iter_content(chunk_size=2048):
            if chunk:
                pdf.write(chunk)


async def job(session, index, url):
    # 声明为异步函数

    file = await session.get(url, verify_ssl=False)
    # 触发到await就切换，等待get到数据
    filecode = await file.read()
    # 读取内容
    with open(f'/tmp/123/{index}.jpg', 'wb') as f:
        # 写入到指定目录下的文件中
        f.write(filecode)
    return str(url)


async def main(loop, URL):
    async with aiohttp.ClientSession() as session:
        # 建立会话session
        tasks = []

        # 建立所有任务
        for index, url in enumerate(URL):
            tasks.append(loop.create_task(job(session, index, url)))

        finished, unfinished = await asyncio.wait(tasks)
        # 触发await，等待任务完成
        # all_results = [r.result() for r in finished]
        # 获取所有结果
        # print("all_results:", all_results)


URL = [
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
    "https://cdn.pixabay.com/photo/2023/04/16/09/49/waterfall-7929685_1280.jpg",
]

t1 = time.time()
# 协程方式3秒以内
loop = asyncio.get_event_loop()
loop.run_until_complete(main(loop, URL))
loop.close()

# 串行20秒以上
# for index, url in enumerate(URL):
#     down_img(url, index)

t2 = time.time()
print("time:", t2 - t1)