aiohttp/asyncio测试代理是否可用
#!/usr/bin/env python # encoding: utf-8 from proxyPool.db import RedisClient import asyncio import aiohttp import time try: from aiohttp import ClientError,client_exceptions except: from aiohttp import ClientProxyConnectionError as ProxyConnectionError VALID_STATUS_CODES=[200] TEST_URL='http://www.baidu.com' BATCH_TEST_SIZE=100 class Tester(object): def __init__(self): self.redis=RedisClient() async def test_single_proxy(self,proxy): """ 测试单个代理 :param proxy:单个代理 :return: None """ conn=aiohttp.TCPConnector(verify_ssl=False) async with aiohttp.ClientSession(connector=conn) as session: try: if isinstance(proxy,bytes): proxy=proxy.decode('utf-8') real_proxy='http://'+proxy print("正在测试",proxy) async with session.get(TEST_URL,proxy=real_proxy,timeout=15) as response: if response.status in VALID_STATUS_CODES: self.redis.max(proxy) print("代理可用",proxy) else: self.redis.decrease(proxy) print("请求响应码不合法",proxy) except(ClientError,client_exceptions.ClientConnectorError,TimeoutError,AttributeError): self.redis.decrease(proxy) print("代理请求失败",proxy) def run(self): """ 测试函数 :return:None """ print("测试器开始运行") try: proxies=self.redis.all() loop=asyncio.get_event_loop() #批量测试 for i in range(0,len(proxies),BATCH_TEST_SIZE): test_proxies=proxies[i:i+BATCH_TEST_SIZE] tasks=[self.test_single_proxy(proxy) for proxy in test_proxies] loop.run_until_complete(asyncio.wait(tasks)) time.sleep(5) except Exception as e: print("测试器发生错误",e.args)