scrapy settings
1 scrapy settings
# -*- coding: utf-8 -*- # Scrapy settings for phone project BOT_NAME = 'phone' SPIDER_MODULES = ['phone.spiders'] NEWSPIDER_MODULE = 'phone.spiders' ROBOTSTXT_OBEY = False DOWNLOAD_DELAY = 3 COOKIES_ENABLED = False SPIDER_MIDDLEWARES = { # 'phone.middlewares.PhoneSpiderMiddleware': 543, } DOWNLOADER_MIDDLEWARES = { # 'phone.middlewares.MyCustomDownloaderMiddleware': 543, } ITEM_PIPELINES = { # 'phone.pipelines.PhonePipeline': 300, } SCHEDULER = "scrapy_redis.scheduler.Scheduler" DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter" REDIS_URL = 'redis://root:123456@58.221.49.25:6379'
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步