异步爬虫实现的多种方式

对1000个url进行请求，并处理响应，测试每一种方法的耗时

一、grequests

单线程+协程

import base64
import time
import grequests
import requests
from io import BytesIO

from rosi.settings import MYSQL_CONFIG, MONGO_CONFIG
from rosi.db.mysqldb import MysqlDB
from rosi.db.mongodb import MongoDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def get_image_b64_by_grequests(url):
    response = requests.get(url)
    b64_data = base64.b64encode(BytesIO(response.content).read())
    return b64_data


def update_mongo_data(data):
    db = MongoDB(**MONGO_CONFIG)
    db.update('rosinew', 'image_data', data)


def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


def callback_func(r, *args, **kwargs):
    """
    回调函数，参数必须这样写
    :param r:
    :param args:
    :param kwargs:
    :return:
    """
    if r is None:
        print('非成功请求')
        return
    b64_data = base64.b64encode(BytesIO(r.content).read())
    b64_data = str(b64_data, 'utf-8')
    update_data(b64_data, r.url)
    print(r.url)

def exception_handler(request, exception):
    """
    异常回调函数
    :param request: 
    :param exception: 
    :return: 
    """
    print('出错了')

def main():
    data = get_data()
    s_time = time.time()
    
    requests_list = (grequests.get(item[1], callback=callback_func) for item in data)
    response_list = grequests.map(requests_list, size=10, exception_handler=exception_handler)
    
    print(response_list)
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main()

结果：

二、多线程+grequests

4个线程，每个线程并发10个协程

import base64
import time
import grequests
import requests
from io import BytesIO
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed

from rosi.settings import MYSQL_CONFIG, MONGO_CONFIG
from rosi.db.mysqldb import MysqlDB
from rosi.db.mongodb import MongoDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def list_of_groups(init_list, children_list_len):
    list_of_groups = zip(*(iter(init_list),) * children_list_len)
    end_list = [list(i) for i in list_of_groups]
    count = len(init_list) % children_list_len
    end_list.append(init_list[-count:]) if count != 0 else end_list
    return end_list


def get_image_b64_by_grequests(url):
    response = requests.get(url)
    b64_data = base64.b64encode(BytesIO(response.content).read())
    return b64_data


def update_mongo_data(data):
    db = MongoDB(**MONGO_CONFIG)
    db.update('rosinew', 'image_data', data)


def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


def callback_func(r, *args, **kwargs):
    """
    回调函数，参数必须这样写
    :param r:
    :param args:
    :param kwargs:
    :return:
    """
    if r is None:
        print('非成功请求')
        return
    b64_data = base64.b64encode(BytesIO(r.content).read())
    b64_data = str(b64_data, 'utf-8')
    update_data(b64_data, r.url)
    print(r.url)


def exception_handler(request, exception):
    """
    异常回调函数
    :param request:
    :param exception:
    :return:
    """
    print('出错了')


def grequests_func(data):
    requests_list = (grequests.get(item[1], callback=callback_func) for item in data)
    response_list = grequests.map(requests_list, size=10, exception_handler=exception_handler)
    print(response_list)
    return


def main():
    data = get_data()
    init_data = list_of_groups(data, 250)
    s_time = time.time()
    # 4个多线程，每个线程并发10个协程
    with ThreadPoolExecutor(max_workers=4) as executor:
        all_task = [executor.submit(grequests_func, (params)) for params in init_data]
        wait(all_task, return_when=ALL_COMPLETED)
    for future in as_completed(all_task):
        item = future.result()
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main()

结果：

 

三、asyncio

 使用python3异步协程asyncio，单线程并发数10

import asyncio
import aiohttp
import base64
import time
from io import BytesIO
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed

from rosi.settings import MYSQL_CONFIG
from rosi.db.mysqldb import MysqlDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


async def fetch(client, url):
    async with client.get(url) as resp:
        # assert resp.status == 200
        content = b''
        while True:
            chunk = await resp.content.read(1024)
            if not chunk:
                break
            content += chunk
        print(url)
        return (content, url)


async def fetch_all(urls):
    async with asyncio.Semaphore(10):
        async with aiohttp.ClientSession() as client:
            return await asyncio.gather(*[fetch(client, url) for url in urls])


def main():
    urls = [item[1] for item in get_data()]
    s_time = time.time()
    loop = asyncio.get_event_loop()
    # 异步获取任务
    results = loop.run_until_complete(fetch_all(urls))
    for result in results:
        content = result[0]
        url = result[1]
        b64_data = base64.b64encode(BytesIO(content).read())
        b64_data = str(b64_data, 'utf-8')
        update_data(b64_data, url)
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main()

 结果：

四、多线程+asyncio

4个线程，每个线程并发10个协程

import asyncio
import aiohttp
import base64
import time
from io import BytesIO
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed

from rosi.settings import MYSQL_CONFIG
from rosi.db.mysqldb import MysqlDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def list_of_groups(init_list, children_list_len):
    list_of_groups = zip(*(iter(init_list),) * children_list_len)
    end_list = [list(i) for i in list_of_groups]
    count = len(init_list) % children_list_len
    end_list.append(init_list[-count:]) if count != 0 else end_list
    return end_list


def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


async def fetch(client, url):
    async with client.get(url) as resp:
        # assert resp.status == 200
        content = b''
        while True:
            chunk = await resp.content.read(1024)
            if not chunk:
                break
            content += chunk
        print(url)
        return (content, url)


async def fetch_all(urls):
    async with asyncio.Semaphore(10):
        async with aiohttp.ClientSession() as client:
            return await asyncio.gather(*[fetch(client, url[1]) for url in urls])


def run(urls):
    loop1 = asyncio.new_event_loop()
    asyncio.set_event_loop(loop1)
    loop = asyncio.get_event_loop()
    # 异步获取任务
    results = loop.run_until_complete(fetch_all(urls))
    for result in results:
        content = result[0]
        url = result[1]
        b64_data = base64.b64encode(BytesIO(content).read())
        b64_data = str(b64_data, 'utf-8')
        update_data(b64_data, url)


def main():
    data = get_data()
    init_data = list_of_groups(data, 250)
    s_time = time.time()
    # 4个多线程，每个线程并发10个协程
    with ThreadPoolExecutor(max_workers=4) as executor:
        all_task = [executor.submit(run, (params)) for params in init_data]
        wait(all_task, return_when=ALL_COMPLETED)
    for future in as_completed(all_task):
        item = future.result()
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main() 

结果：

 asyncio协程+多线程没啥提升，还不如只用asyncio

五、tornado异步

tornado官方文档中的示例改了一下，并发10协程

import time
import base64
from datetime import timedelta
from io import BytesIO
from tornado import httpclient, gen, ioloop, queues

from rosi.settings import MYSQL_CONFIG
from rosi.db.mysqldb import MysqlDB

try:
    from HTMLParser import HTMLParser
    from urlparse import urljoin, urldefrag
except ImportError:
    from html.parser import HTMLParser
    from urllib.parse import urljoin, urldefrag


concurrency = 10


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


@gen.coroutine
def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


@gen.coroutine
def get_request_async(url):
    try:
        response = yield httpclient.AsyncHTTPClient().fetch(url)
        image_data = base64.b64encode(BytesIO(response.body).read())
    except Exception as e:
        print('Exception: %s %s' % (e, url))
        raise gen.Return([])
    raise gen.Return(image_data)


@gen.coroutine
def main():
    q = queues.Queue()
    # 添加url
    for item in get_data():
        q.put(item[1])
    start = time.time()
    print(q.qsize())
    @gen.coroutine
    def fetch_url():
        current_url = yield q.get()
        try:
            b64_data = yield get_request_async(current_url)
            image_data = str(b64_data, 'utf-8')
            yield update_data(image_data, current_url)
            print(current_url)
        except Exception as e:
            print('**************************************', str(e))
        finally:
            q.task_done()

    @gen.coroutine
    def worker():
        while True:
            yield fetch_url()

    for _ in range(concurrency):
        worker()
    yield q.join(timeout=timedelta(seconds=3000))
    print(q.qsize())
    print(f'结束了： {time.time() - start}')


if __name__ == '__main__':
    import logging
    logging.basicConfig()
    io_loop = ioloop.IOLoop.current()
    io_loop.run_sync(main)

结果：

六、多线程

10个线程测试一下，多进程就不测了

import base64
import requests
import time
from io import BytesIO
from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED, as_completed

from rosi.settings import MYSQL_CONFIG
from rosi.db.mysqldb import MysqlDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def update_data(data, url):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where image_url='{}'".format(data, url)
    db.update(sql)


def get_image_b64_by_requests(url):
    response = requests.get(url)
    b64_data = base64.b64encode(BytesIO(response.content).read())
    return b64_data


def deal_func(item):
    b64_data = get_image_b64_by_requests(item[1])
    b64_data = str(b64_data, 'utf-8')
    update_data(b64_data, item[1])
    print(item[1])
    return None


def main():
    data = get_data()
    s_time = time.time()
    with ThreadPoolExecutor(max_workers=10) as executor:
        all_task = [executor.submit(deal_func, (item)) for item in data]
        wait(all_task, return_when=ALL_COMPLETED)
    for future in as_completed(all_task):
        item = future.result()
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main()

 结果：

七、同步方式

测试一下同步的方式，用作对比

import base64
import time
import requests
from io import BytesIO
from rosi.settings import MYSQL_CONFIG
from rosi.db.mysqldb import MysqlDB


def get_data():
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "select * from rosi where image_b64=''"
    data = db.find(sql, limit=1000)
    print(len(data))
    return data


def get_image_b64_by_requests(url):
    response = requests.get(url)
    b64_data = base64.b64encode(BytesIO(response.content).read())
    return b64_data


def update_data(data, id):
    db = MysqlDB(**MYSQL_CONFIG)
    sql = "update rosi set image_b64='{}' where id='{}'".format(str(data, 'utf-8'), id)
    db.update(sql)


def main():
    data = get_data()
    s_time = time.time()
    for item in data:
        try:
            image = get_image_b64_by_requests(item[1])
            update_data(image, item[0])
            print(item[2],item[4])
        except Exception as e:
            print('*****************', str(e))
            continue
    print(f'结束了： {time.time() - s_time}')


if __name__ == '__main__':
    main()

结果：