python批量下载

以下载图片为例：

from gevent import monkey

monkey.patch_all()
from gevent.pool import Pool
from urllib.parse import urlparse
import requests
import sys
import os


def download(url):
    chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
    headers = {'User-Agent': chrome}
    r = requests.get(url.strip(), headers=headers, stream=True)

    output_dir = "pic"  # 图片存放文件夹
    # 判断输出文件夹是否存在。不存在就创建
    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)

    url_parse = urlparse(url)  # 图片来源url
    fname = os.path.split(url_parse.path)[1]  # 图片名
    filename = output_dir + os.sep + fname  # 图片存放位置
    print("开始下载：%s" % fname)
    with open(filename, 'wb')as f:
        f.write(r.content)  # 保存图片
        print("下载完成：%s" % fname)


if __name__ == "__main__":
    if len(sys.argv) == 2:
        filename = sys.argv[1]
        f = open(filename, "r")
        p = Pool(100)
        for line in f.readlines():
            if line:
                p.spawn(download, line.strip())
                f.close()
                p.join()
    else:
        print('Usage: python %s urls.txt' % sys.argv[0])

　　

 

python  downloader.py test.txt