使用concurrent.futures和ProcessPoolExecutor来替代线程和进程
concurrent.futures和ProcessPoolExecutor这两个类实现的借口分别在不同的线程或进程中执行可调用的对象,这两个类在内部维护者一个工作线程或进程池,以及要执行的队列,这两个借口抽象的层级很高,无需关注实现细节
普通方法实现下载国旗
import os import time import sys import requests POP20_CC=('CN IN US ID BR PK NG BD RU JP' 'MX PH VN ET EG DE IR TR CD FR').split() BASE_URL='http://flupy.org/data/flags' DEST_DIR='downloads/' def save_flag(img,filename): path=os.path.join(DEST_DIR,filename) with open(path,'wb')as fp: fp.write(img) def get_flag(cc): url='{}/{cc}/{cc}.gif'.format(BASE_URL,cc=cc.lower()) resp=requests.get(url) return resp.content def show(text): print(text,end="") sys.stdout.flush() def download_many(cc_list): for cc in sorted(cc_list): image=get_flag(cc) show(cc) save_flag(image,cc.lower()+'.gif') return len(cc_list) def main(download_many): to=time.time() count=download_many(POP20_CC) elapsed=time.time() msg='\n{} flags downloaded in {:.2f}s' print(msg.format(count,elapsed)) if __name__=='__main__': main(download_many)
替代多线程方法
#!/usr/bin/env python # -*- coding:utf-8 -*- from concurrent import futures import sys # sys.path.append(r"F:\regtest\asyncio!") # from qw import save_flag,get_flag,show ,main import os import time import sys MAX_WORKERS=20 import requests POP20_CC=('CN IN US ID BR PK NG BD RU JP' 'MX PH VN ET EG DE IR TR CD FR').split() BASE_URL='http://flupy.org/data/flags' DEST_DIR='downloads/' def save_flag(img,filename): path=os.path.join(DEST_DIR,filename) with open(path,'wb')as fp: fp.write(img) def get_flag(cc): url='{}/{cc}/{cc}.gif'.format(BASE_URL,cc=cc.lower()) resp=requests.get(url) return resp.content def show(text): print(text,end="") sys.stdout.flush() def download_one(cc): image=get_flag(cc) show(cc) save_flag(image,cc.lower()+'.gif') return cc def download_many(cc_list): workers=min(MAX_WORKERS,len(cc_list)) with futures.ThreadPoolExecutor(workers) as executor: res=executor.map(download_one,sorted(cc_list)) return len(list(res)) def main(download_many): to=time.time() count=download_many(POP20_CC) elapsed=time.time() msg='\n{} flags downloaded in {:.2f}s' print(msg.format(count,elapsed)) if __name__=='__main__': main(download_many)
有了这个神器就再也不用耗费精力去创建线程池或者进程池,以及队列来处理问题了。~
关于如何创建线程池进程池,在python基础篇~