批量异步上传aws图片脚本（python）

背景

工作中需要上传一些测试图片，于是网上找找资料（官方说明），前置步骤如下。

1. python需要3.8以上，安装最新的boto3库：

   pip install boto3

2. 有一个S3权限的aws账户，得到访问密钥ACCESS_KEY与SECRET_KEY，以及上传图片的存储桶位置
3. 安装异步编程asyncio，aiohttp库，方便本地异步上传图片

代码实现

# -*- coding: utf-8 -*-
"""
# @Author : qgc
# @Time : 2024/3/13 14:28
# @File : upload_to_aws.py
# Description : 文件说明
"""
import os
import boto3
import asyncio
import aiohttp

aws_access_key_id = 'xxxxxx'
aws_secret_access_key = 'xxxxxxx'
bucket_name = 'deme-normal'   # 存储桶的具体路由，路由不存在会上传失败，但不会报错

async def upload_to_aws(local_file, s3_file):
    s3 = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
    try:
        s3.upload_file(local_file, bucket_name, s3_file)
        print(f"{local_file} upload to s3://{bucket_name}/{s3_file} successed")
        return True
    except FileNotFoundError:
        print('本地文件未找到')
        return False

async def upload_task(url, session, semaphore):
    try:
        if url:
            local_filename = url
            s3_path = 'qgctest/' + os.path.basename(url)
            await upload_to_aws(local_filename, s3_path)
    except Exception as e:
        print(f'处理{url}:{e}')

def get_image_files(directory):
    image_files = []
    if directory:
        for file in os.listdir(directory):
            if file.endswith('.jpg') or file.endswith('.jpeg') or file.endswith('.png') or file.endswith('.gif'):
                image_files.append(os.path.join(directory, file))
    return image_files

async def main(image_files):
    semaphore = asyncio.Semaphore(10)
    connector = aiohttp.TCPConnector(limit=10)
    async with aiohttp.ClientSession(connector=connector) as session:
        tasks = [upload_task(url, session, semaphore) for url in image_files]
        await asyncio.gather(*tasks)


if __name__ == '__main__':
    directory = f'E:\图片\images2'  # 本地图片路径
    image_files = get_image_files(directory)
    # image_files = ['E:\\图片\\images2\\0.jpg']
    print(f'共找到{len(image_files)}个图片')
    asyncio.get_event_loop().run_until_complete(main(image_files))