scrapy 2.x相关配置
-
cryptography==38.0.4
pyopenssl==22.0.0
certifi==2023.5.7
-
在windows python3.8+下的scrapy框架内使用asyncio需要先执行以下代码:
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-
# 开启协程相关配置
import platform
import asyncio
if platform.system().lower() == 'windows': # windows配置
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # win下python3.8+需要的特殊配置
elif platform.system().lower() == 'linux': # linux配置
import scrapy
scrapy.utils.reactor.install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')