【Python】new【容器 | 迭代对象 | 迭代器 | 生成器 | 生成器表达式 | 协程 | 期物 | 任务】

 

"""
# 例子 有些时候我保存在博客园的资源图片地址会失效，下面这个例子是最近一次生效的，再之后是之前的

from enum import Enum
from collections import namedtuple
from concurrent import futures
from tqdm import tqdm
import os
import time
import sys
import requests
import collections
import asyncio

Result = namedtuple('Result', 'status cc')
HTTPStatus = Enum('Status', 'ok not_found error')

BASE_URL = 'https://images2018.cnblogs.com/blog/1239321/201808'

DEST_DIR = 'downloads'

MAX_WORKERS = 20

POP20_CC = 'aaaa1239321-20180802153722539-931669752 bbbbb1239321-20180802154629131-632652627 1239321-20180802153722539-931669752'.split()

def save_flag(img, filename):
    path = os.path.join(sys.path[0], DEST_DIR, filename)
    with open(path + '.PNG', 'wb') as fp:
        fp.write(img)

def get_flag(cc):
    url = '{}/{cc}.png'.format(BASE_URL, cc=cc)
    resp = requests.get(url)
    if 'PNG' in resp.text:
        return resp.content
    elif 'not found' in resp.text or 'not exist' in resp.text:
        resp.status_code = 404
        resp.raise_for_status()

def download_one(cc, verbose=False):
    try:
        image = get_flag(cc)
    except requests.exceptions.HTTPError as exc:
        res = exc.response
        if res.status_code == 404:
            status = HTTPStatus.not_found
            msg = 'not found'
            res.reason = 'NOT FOUND'
            raise
        else:
            raise
    else:
        save_flag(image, cc)
        status = HTTPStatus.ok
        msg = 'OK'
    if verbose:
        print (cc, msg)
    return Result(status, cc)

def download_many(cc_list, verbose):
    counter = collections.Counter()
    with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        to_do_map = {}
        for cc in sorted(cc_list):
            future = executor.submit(download_one, cc, verbose)
            to_do_map[future] = cc
        done_iter = futures.as_completed(to_do_map)
        if verbose:
            done_iter = tqdm(done_iter, total=len(cc_list))
        for future in done_iter:
            try:
                res = future.result()
            except requests.exceptions.HTTPError as exc:
                error_msg = 'HTTP{res.status_code} - {res.reason}'
                error_msg = error_msg.format(res=exc.response)
            except requests.exceptions.ConnectionError as exc:
                error_msg = 'Connection error'
            else:
                error_msg = ''
                status = res.status
            if error_msg:
             status = HTTPStatus.error
             counter[status] += 1
            if not verbose and error_msg:
             print ('*** error for {}:{}'.format(to_do_map[future], error_msg))
    return counter

def main():
    t0 = time.time()
    count = download_many(POP20_CC, verbose=False)
    elapsed = time.time() - t0
    msg = '\n{} flags downloaded in {:.2f}s'
    print(msg.format(count, elapsed))

if __name__ == '__main__':
    main()

# 栗子17-3 使用futures.ThreadPoolExecutor类实现多线程下载的脚本

import os
import sys
import time
import requests
from concurrent import futures

BASE_URL = 'https://images2018.cnblogs.com/blog/1239321/201808'

POP20_CC = '1239321-20180808065117364-1539273796 1239321-20180808065129112-103367989 1239321-20180808065422695-222625730 1239321-20180808065410900-539910454'.split()

DEST_DIR = 'downloads'

MAX_WORKERS = 20

def save_flag(img, filename):
    path = os.path.join(sys.path[0], DEST_DIR, filename)
    with open(path, 'wb') as fp:
        fp.write(img)

def get_flag(cc):
    url = '{}/{cc}.png'.format(BASE_URL, cc=cc.lower())
    resp = requests.get(url)
    return resp.content

def show(text):
    print (text+'\n', end='')
    sys.stdout.flush()

def download_one(cc):
    image = get_flag(cc)
    show(cc)
    save_flag(image, cc.lower() + '.PNG')
    return cc

def download_many(cc_list):
    workers = min(MAX_WORKERS, len(cc_list))
    with futures.ThreadPoolExecutor(workers) as executor:
        res = executor.map(download_one, sorted(cc_list))
    return len(list(res))

def main(download_many):
    t0 = time.time()
    count = download_many(POP20_CC)
    elapsed = time.time() - t0
    msg = '\n{} flag downloaded in {:.2f}s'
    print (msg.format(count, elapsed))

if __name__ == '__main__':
    main(download_many)


# 栗子 17-4 了解期物未何物
import os
import sys
import time
import requests
from concurrent import futures

BASE_URL = 'https://images2018.cnblogs.com/blog/1239321/201808'

POP20_CC = '1239321-20180808065117364-1539273796 1239321-20180808065129112-103367989 1239321-20180808065422695-222625730 1239321-20180808065410900-539910454'.split()

DEST_DIR = 'downloads'

MAX_WORKERS = 20

def save_flag(img, filename):
    path = os.path.join(sys.path[0], DEST_DIR, filename)
    with open(path, 'wb') as fp:
        fp.write(img)

def get_flag(cc):
    url = '{}/{cc}.png'.format(BASE_URL, cc=cc.lower())
    resp = requests.get(url)
    return resp.content

def show(text):
    print (text+'\n', end='')
    sys.stdout.flush()

def download_one(cc):
    image = get_flag(cc)
    show(cc)
    save_flag(image, cc.lower() + '.PNG')
    return cc

def download_many(cc_list):
    cc_list = cc_list[:5]
    with futures.ThreadPoolExecutor(max_workers=3) as executor:
        to_do = []
        for cc in sorted(cc_list):
            future = executor.submit(download_one, cc)
            to_do.append(future)
            msg = 'Scheduled for {}:{}'
            print (msg.format(cc, future))
        results = []
        for future in futures.as_completed(to_do):
            res = future.result()
            msg = '{} result: {!s}'
            print (msg.format(future, res))
            results.append(res)
    return len(results)


def main(download_many):
    t0 = time.time()
    count = download_many(POP20_CC)
    elapsed = time.time() - t0
    msg = '\n{} flag downloaded in {:.2f}s'
    print (msg.format(count, elapsed))

if __name__ == '__main__':
    main(download_many)

"""
# 栗子18-1 threading