Python - aiohttp
学习资料
1.https://github.com/aio-libs/aiohttp
2.官网：https://docs.aiohttp.org/en/stable/
3.https://www.inte.net/news/275463.html
基本使用
"""
- event_loop: 事件循环，可以把一些函数注册到这个事件循环上，当满足条件发生的时候，就调用相应的处理方法
- coroutine: Python中代指协程对象类型，可以将协程对象注册到事件循环上。可以使用async 关键字定义一个方法，这个方法
在调用的时候不会立即执行，而是会返回一个协程对象
- task: 协程对象的进行一步封装，包含协程对象的各个状态
- await关键字: 可以将耗时等待的操作挂起，让出控制权，如果协程在执行的时候遇到await，事件循环就会将本协程挂起，转而执行
别的协程，直到其他协程挂起或执行完毕
"""
import asyncio
async def execture(x):
print(f'Number:{x}')
coroutine = execture(1)
print(f'coroutine called after')
print(f'type:{type(coroutine)}')
loop = asyncio.get_event_loop()
loop.run_until_complete(coroutine)
#out:
"""
E:\PyProject\study\test_aiohttp.py:69: DeprecationWarning: There is no current event loop
loop = asyncio.get_event_loop()
coroutine called after
type:<class 'coroutine'>
Number:1
"""
从以上结果可以看出调用async 定义的方法返回的是coroutine 类型，方法本身并没有执行，
在注册到事件监听之后才会执行
import asyncio
async def execture(x):
print(f'Number:{x}')
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
task = asyncio.ensure_future(execture(1),loop=loop) # 注册后返回task
print(f'type task is :{type(task)}')
print(f'task run before:{task}')
loop.run_until_complete(task)
print(f'task run after:{task}')
out:
可以看到task的状态信息：
import aiohttp
import asyncio
async def fetch(session,url):
async with session.get(url) as resp: #resp = await session.get(url)
return await resp.text(), resp.status
async def main():
async with aiohttp.ClientSession() as session:
html,status = await fetch(session, 'https://www.cnblogs.com/czzz/')
print(f'html:{html}')
print(f'status:{status}')
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
参数设置
import asyncio
import aiohttp
async def main():
params = {'name': 'sun', 'age': 25}
async with aiohttp.ClientSession() as session:
async with session.get('http://www.httpbin.org/get', params = params) as response:
print(await response.text())
POST 请求
# POST 表单请求
async def main():
data = {'name': 'sun', 'age': 25}
async with aiohttp.ClientSession() as session:
async with session.post('https://www.httpbin.org/post', data=data) as response:
print(await response.text())
# POST JSON请求体
async def main():
json_data = {'name': 'sun', 'age': 25}
async with aiohttp.ClientSession() as session:
async with session.get('http://www.httpbin.org/get', json = json_data) as response:
print(await response.text())
ClientSession
TCPConnector
ClientTimeout
并发限制
aiohttp可支持的并发量非常高，几万，几十万，几百万都是可以的，但是目标网站可能无法在短时间响应，而且有
可能把目标网站爬挂的风险，因此需要控制下爬取的并发量
import asyncio
import aiohttp
concurrency = 5
url = 'https://www.baidu.com'
semaphore = asyncio.Semaphore(concurrency)
session = None
async def scrape_api():
async with semaphore:
print(f'url:{url}')
async with session.get(url) as resp:
await asyncio.sleep(1)
return await resp.text()
async def main():
global session
session = aiohttp.ClientSession()
tasks = [asyncio.ensure_future(scrape_api()) for _ in range(10)]
await asyncio.gather(*tasks)
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())
# out:
'''
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
url:https://www.baidu.com
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x000001C7FEE34640>
Unclosed connector
connections: ['[(<aiohttp.client_proto.ResponseHandler object at 0x000001C7FEE6AD40>, 43780.609), (<aiohttp.client_proto.ResponseHandler object at 0x000001C7FEE68400>, 43780.609), (<aiohttp.client_proto.ResponseHandler object at 0x000001C7FEE6B340>, 43780.609), (<aiohttp.client_proto.ResponseHandler object at 0x000001C7FEE6B100>, 43780.609), (<aiohttp.client_proto.ResponseHandler object at 0x000001C7FEE6AEC0>, 43780.609)]']
connector: <aiohttp.connector.TCPConnector object at 0x000001C7FEE35960>
'''
异步爬虫
案例1：网址- https://spa5.scrape.center/
# @Time: 2022/11/21 22:46
# @Authot: zzc
# @Project: asyncioDemo
# @File:demo1.py
import json
import aiohttp
import asyncio
import logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s: %(message)s')
INDEX_URL = 'Https://spa5.scrape.center/api/book/?limit=18&offset={offset}'
DETIAL_UTL = 'https://spa5.scrape.center/api/book/{id}/'
PAGE_SIZE = 18
PAGE_NUMBER = 3
CONCURRENCY = 3
semaphone = asyncio.Semaphore(CONCURRENCY) # 信号标
session = None
# 通用爬取方法
async def scrape_api(url): # url 会变
async with semaphone:
try:
logging.info(f'scraping {url}')
async with session.get(url) as resp:
data = await resp.json()
print(data)
return data
except aiohttp.ClientError:
logging.error(f'error occurred while scraping {url}', exc_info=True)
# 爬取列表页
async def scrape_index(page): # page 由 main传过来
# 计算offset
# 格式化url，调用api
url = INDEX_URL.format(offset=PAGE_SIZE * (page - 1))
return await scrape_api(url)
async def main():
global session
session = aiohttp.ClientSession()
# 生成task
tasks = [asyncio.ensure_future(scrape_index(page)) for page in range(1, PAGE_NUMBER + 1)]
results = await asyncio.gather(*tasks) # 返回值是 将单个scrape_api 的返回值组装成list
print(results)
logging.info('results : %s', json.dumps(results, ensure_ascii=False, indent=2))
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(main())
#out:
'''
{'count': 9040, 'results': [{'id': '7952978', 'name': 'Wonder', 'authors': ['R. J. Palacio'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s27252687.jpg', 'score': '8.8'}, {'id': '7916054', 'name': '清白家风', 'authors': ['\n 董桥', '海豚简装'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s27250764.jpg', 'score': '7.5'}, {'id': '7698729', 'name': '法老的宠妃 终结篇（上下册）', 'authors': ['\n 悠世', '法老的宠妃'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s7027218.jpg', 'score': '7.2'}, {'id': '7658805', 'name': '士为知己（全二册）', 'authors': ['蓝色狮'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8866404.jpg', 'score': '7.7'}, {'id': '7564736', 'name': '那些年，我们一起追的女孩', 'authors': ['\n 九把刀', '九把刀作品集·现代版'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7049425.jpg', 'score': '8.2'}, {'id': '7440370', 'name': '非我倾城（全三册）', 'authors': ['\n 墨舞碧歌'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8916163.jpg', 'score': '7.8'}, {'id': '7163250', 'name': '明朝那些事儿', 'authors': ['\n 当年明月'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s29399938.jpg', 'score': '9.2'}, {'id': '7154825', 'name': '我和你的笑忘书', 'authors': ['夏七夕'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7028274.jpg', 'score': '6.4'}, {'id': '7154690', 'name': '王小波全集 第一卷', 'authors': ['\n 王小波', '王小波全集（凤凰壹力2012版）'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s11315662.jpg', 'score': '8.3'}, {'id': '7154675', 'name': '怦然心动', 'authors': ['\n [美]\n 文德琳·范·德拉安南'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s7026741.jpg', 'score': '8.8'}, {'id': '7153409', 'name': '龙枪编年史（全3册）', 'authors': ['[美] 玛格丽特·魏丝', '[美] 崔西·西克曼'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s9100391.jpg', 'score': '8.8'}, {'id': '7067986', 'name': '龙枪传奇（全三册）', 'authors': ['玛格丽特·魏丝 崔西·西克曼'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s10423583.jpg', 'score': '9.1'}, {'id': '7067983', 'name': '黎明之街', 'authors': ['\n [日]\n 东野圭吾', '新经典文化', '新经典文库·东野圭吾作品'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8871926.jpg', 'score': '7.2'}, {'id': '7067149', 'name': '认知心理学及其启示', 'authors': ['约翰•R•安德森'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7059106.jpg', 'score': '8.8'}, {'id': '7065529', 'name': '银河帝国2：基地与帝国', 'authors': ['\n [美]\n 艾萨克·阿西莫夫', '读客文化', '银河帝国'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s9117163.jpg', 'score': '9.0'}, {'id': '7065521', 'name': '银河帝国：基地', 'authors': ['\n [美]\n 艾萨克·阿西莫夫', '读客文化', '银河帝国'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8973055.jpg', 'score': '9.0'}, {'id': '7063979', 'name': '小学教材全解-四年级语文下', 'authors': ['\n 薛金星'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s24506696.jpg', 'score': '7.3'}, {'id': '7062629', 'name': '越界言论（第3卷）', 'authors': ['许子东'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s22696330.jpg', 'score': '8.3'}]}
{'count': 9040, 'results': [{'id': '7062628', 'name': '许子东讲稿-张爱玲.郁达夫.香港文学-卷二', 'authors': ['\n 许子东', '许子东讲稿'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7034934.jpg', 'score': '8.4'}, {'id': '7060186', 'name': 'GOTH断掌事件', 'authors': ['\n [日]\n 乙一', '乐读文库'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s7656327.jpg', 'score': '8.2'}, {'id': '7059909', 'name': '南音 （上）', 'authors': ['\n 笛安', '“龙城”系列'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s7035923.jpg', 'score': '7.4'}, {'id': '7058564', 'name': '第一辩手5·星河的泪痕', 'authors': ['双子星罗'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7019904.jpg', 'score': '6.8'}, {'id': '7058461', 'name': '手绢上的花田', 'authors': ['(日)安房直子'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s28798515.jpg', 'score': '9.2'}, {'id': '7058064', 'name': '銀之匙 01', 'authors': ['\n [日]\n 荒川弘', '銀之匙'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7051564.jpg', 'score': '9.3'}, {'id': '7057385', 'name': '听幾米唱歌', 'authors': ['几米 绘'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s8857547.jpg', 'score': '8.5'}, {'id': '7057187', 'name': '陶庵梦忆', 'authors': ['张岱'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8859312.jpg', 'score': '9.5'}, {'id': '7056685', 'name': '天黑以后', 'authors': ['\n [日]\n 村上春树', '新经典文化', '\n 施小炜', '新经典文库·村上春树作品'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8913032.jpg', 'score': '7.5'}, {'id': '7056649', 'name': '歧路', 'authors': ['\n 脉脉', '\n 渥丹'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7034174.jpg', 'score': '8.6'}, {'id': '7055441', 'name': '罗马人的故事1', 'authors': ['盐野七生'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s8488837.jpg', 'score': '8.8'}, {'id': '7053013', 'name': '魔法科高校の劣等生〈3〉', 'authors': ['佐島 勤'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7046464.jpg', 'score': '7.9'}, {'id': '7047308', 'name': '查理九世10·最后的古寺神佛', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8857032.jpg', 'score': '7.7'}, {'id': '7047307', 'name': '查理九世12·失落的海底城', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7652904.jpg', 'score': '7.8'}, {'id': '7047305', 'name': '查理九世11·冥府之船', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7652905.jpg', 'score': '8.5'}, {'id': '7015296', 'name': '最美遇见你', 'authors': ['\n 顾西爵'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8486741.jpg', 'score': '7.4'}, {'id': '7008029', 'name': '指环王1-3合集(3DVD9院线版)', 'authors': [], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s33456646.jpg', 'score': '9.4'}, {'id': '7007774', 'name': '没有意义就没有摇摆', 'authors': ['\n [日]\n 村上春树', '\n 林少华', '村上春树随笔系列'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s9012766.jpg', 'score': '7.7'}]}
gather:===>[{'count': 9040, 'results': [{'id': '7952978', 'name': 'Wonder', 'authors': ['R. J. Palacio'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s27252687.jpg', 'score': '8.8'}, {'id': '7916054', 'name': '清白家风', 'authors': ['\n 董桥', '海豚简装'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s27250764.jpg', 'score': '7.5'}, {'id': '7698729', 'name': '法老的宠妃 终结篇（上下册）', 'authors': ['\n 悠世', '法老的宠妃'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s7027218.jpg', 'score': '7.2'}, {'id': '7658805', 'name': '士为知己（全二册）', 'authors': ['蓝色狮'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8866404.jpg', 'score': '7.7'}, {'id': '7564736', 'name': '那些年，我们一起追的女孩', 'authors': ['\n 九把刀', '九把刀作品集·现代版'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7049425.jpg', 'score': '8.2'}, {'id': '7440370', 'name': '非我倾城（全三册）', 'authors': ['\n 墨舞碧歌'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8916163.jpg', 'score': '7.8'}, {'id': '7163250', 'name': '明朝那些事儿', 'authors': ['\n 当年明月'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s29399938.jpg', 'score': '9.2'}, {'id': '7154825', 'name': '我和你的笑忘书', 'authors': ['夏七夕'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7028274.jpg', 'score': '6.4'}, {'id': '7154690', 'name': '王小波全集 第一卷', 'authors': ['\n 王小波', '王小波全集（凤凰壹力2012版）'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s11315662.jpg', 'score': '8.3'}, {'id': '7154675', 'name': '怦然心动', 'authors': ['\n [美]\n 文德琳·范·德拉安南'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s7026741.jpg', 'score': '8.8'}, {'id': '7153409', 'name': '龙枪编年史（全3册）', 'authors': ['[美] 玛格丽特·魏丝', '[美] 崔西·西克曼'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s9100391.jpg', 'score': '8.8'}, {'id': '7067986', 'name': '龙枪传奇（全三册）', 'authors': ['玛格丽特·魏丝 崔西·西克曼'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s10423583.jpg', 'score': '9.1'}, {'id': '7067983', 'name': '黎明之街', 'authors': ['\n [日]\n 东野圭吾', '新经典文化', '新经典文库·东野圭吾作品'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8871926.jpg', 'score': '7.2'}, {'id': '7067149', 'name': '认知心理学及其启示', 'authors': ['约翰•R•安德森'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7059106.jpg', 'score': '8.8'}, {'id': '7065529', 'name': '银河帝国2：基地与帝国', 'authors': ['\n [美]\n 艾萨克·阿西莫夫', '读客文化', '银河帝国'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s9117163.jpg', 'score': '9.0'}, {'id': '7065521', 'name': '银河帝国：基地', 'authors': ['\n [美]\n 艾萨克·阿西莫夫', '读客文化', '银河帝国'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s8973055.jpg', 'score': '9.0'}, {'id': '7063979', 'name': '小学教材全解-四年级语文下', 'authors': ['\n 薛金星'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s24506696.jpg', 'score': '7.3'}, {'id': '7062629', 'name': '越界言论（第3卷）', 'authors': ['许子东'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s22696330.jpg', 'score': '8.3'}]}, {'count': 9040, 'results': [{'id': '7062628', 'name': '许子东讲稿-张爱玲.郁达夫.香港文学-卷二', 'authors': ['\n 许子东', '许子东讲稿'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7034934.jpg', 'score': '8.4'}, {'id': '7060186', 'name': 'GOTH断掌事件', 'authors': ['\n [日]\n 乙一', '乐读文库'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s7656327.jpg', 'score': '8.2'}, {'id': '7059909', 'name': '南音 （上）', 'authors': ['\n 笛安', '“龙城”系列'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s7035923.jpg', 'score': '7.4'}, {'id': '7058564', 'name': '第一辩手5·星河的泪痕', 'authors': ['双子星罗'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7019904.jpg', 'score': '6.8'}, {'id': '7058461', 'name': '手绢上的花田', 'authors': ['(日)安房直子'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s28798515.jpg', 'score': '9.2'}, {'id': '7058064', 'name': '銀之匙 01', 'authors': ['\n [日]\n 荒川弘', '銀之匙'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7051564.jpg', 'score': '9.3'}, {'id': '7057385', 'name': '听幾米唱歌', 'authors': ['几米 绘'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s8857547.jpg', 'score': '8.5'}, {'id': '7057187', 'name': '陶庵梦忆', 'authors': ['张岱'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8859312.jpg', 'score': '9.5'}, {'id': '7056685', 'name': '天黑以后', 'authors': ['\n [日]\n 村上春树', '新经典文化', '\n 施小炜', '新经典文库·村上春树作品'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8913032.jpg', 'score': '7.5'}, {'id': '7056649', 'name': '歧路', 'authors': ['\n 脉脉', '\n 渥丹'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7034174.jpg', 'score': '8.6'}, {'id': '7055441', 'name': '罗马人的故事1', 'authors': ['盐野七生'], 'cover': 'https://img1.doubanio.com/view/subject/l/public/s8488837.jpg', 'score': '8.8'}, {'id': '7053013', 'name': '魔法科高校の劣等生〈3〉', 'authors': ['佐島 勤'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7046464.jpg', 'score': '7.9'}, {'id': '7047308', 'name': '查理九世10·最后的古寺神佛', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8857032.jpg', 'score': '7.7'}, {'id': '7047307', 'name': '查理九世12·失落的海底城', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7652904.jpg', 'score': '7.8'}, {'id': '7047305', 'name': '查理九世11·冥府之船', 'authors': ['\n 雷欧幻像', '查理九世'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s7652905.jpg', 'score': '8.5'}, {'id': '7015296', 'name': '最美遇见你', 'authors': ['\n 顾西爵'], 'cover': 'https://img3.doubanio.com/view/subject/l/public/s8486741.jpg', 'score': '7.4'}, {'id': '7008029', 'name': '指环王1-3合集(3DVD9院线版)', 'authors': [], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s33456646.jpg', 'score': '9.4'}, {'id': '7007774', 'name': '没有意义就没有摇摆', 'authors': ['\n [日]\n 村上春树', '\n 林少华', '村上春树随笔系列'], 'cover': 'https://img9.doubanio.com/view/subject/l/public/s9012766.jpg', 'score': '7.7'}]}]
'''
asyncio.wait() 和 async.gather() 的区别?
https://blog.csdn.net/weixin_44818729/article/details/107102593
https://www.jianshu.com/p/6872bf356af7
asyncio: Task, create_task, ensure_future 都可以创建任务，该用哪个？
https://www.cnblogs.com/andy0816/p/15591485.html
asyncio中get_running_loop和get_event_loop的区别
http://30daydo.com/article/613
问题
aiohttp.client_exceptions.ClientOSError: [WinError 121] 信号灯超时时间已到
解决：https://blog.csdn.net/Yooyi_xin/article/details/119698766
aiohttp.client_exceptions.ServerDisconnectedError: Server disconnected
# 解决：
# 1.aiohttp.TCPConnector(limit=0) => aiohttp.TCPConnector(limit=10)
# 2.添加 timeout=aiohttp.ClientTimeout(total=600)
async def get_data_main():
# 初始化数据库
# db_utils.init_db('auto_search_uat.db')
set_logging.set_get_data_log()
db_utils.clear_data('auto_search_uat.db')
# 计算code_info的总页数
try:
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=12), timeout=aiohttp.ClientTimeout(total=600),headers=header) as session:
get_data_api = GetDataApi()
total_pages = get_data_api.get_total_pages()
logger.info(f'total_pages:{total_pages}')
# taks1 = [asyncio.ensure_future(get_code_info(page, session)) for page in range(1, total_pages[1] + 1)]
# taks2 = [asyncio.ensure_future(get_serial_info(page, session)) for page in range(1, total_pages[0] + 1)]
# taks3 = [asyncio.ensure_future(get_product_model(page, session)) for page in range(1, total_pages[2] + 1)]
taks4 = [asyncio.ensure_future(get_keywork(page, session)) for page in range(1, total_pages[3] + 1)]
# taks1.extend(taks2)
# taks1.extend(taks3)
# taks1.extend(taks4)
await asyncio.gather(*taks4)
except ContentTypeError as e:
logger.exception(e)
参考:
1.https://blog.csdn.net/zhb_feng/article/details/118081444
raise ContentTypeError(
aiohttp.client_exceptions.ContentTypeError: 0, message='Attempt to decode JSON with unexpected mimetype: ', url=URL('https://test.jlcerp.com/faManage/searchDataProvider/keywords?pageSize=15&pageNum=2018')
解决：https://qa.1r1g.com/sf/ask/3418826491/
https://cloud.tencent.com/developer/ask/sof/707855/answer/1041708
更新Cookie
https://www.wenjiangs.com/doc/aiohttp-abstractbaseclasses
https://blog.csdn.net/qq_27371025/article/details/124440185
https://vimsky.com/examples/detail/python-ex-aiohttp-CookieJar-update_cookies-method.html
https://cloud.tencent.com/developer/ask/sof/937613/answer/1334367
requests:
http://www.itfanr.cc/2022/10/13/python-requests-use-cookies/
https://blog.csdn.net/lymmurrain/article/details/109037460?utm_medium=distribute.pc_relevant.none-task-blog-2defaultbaidujs_baidulandingword~default-0-109037460-blog-126166367.235v43pc_blog_bottom_relevance_base8&spm=1001.2101.3001.4242.1&utm_relevant_index=3
本文来自博客园，作者：chuangzhou，转载请注明原文链接：https://www.cnblogs.com/czzz/p/16909498.html
