<5> 分布式实现

"""
    配置 redis
    安装 pip3 install scrapy-redis
    修改scrapy项目(先正常实现scrapy爬虫):
"""

# ----1 导入分布式爬虫类
from scrapy_ redis.spiders import RedisSpider, RedisCrawlSpider
# ----2 继承分布式爬虫类
class BookSpider(RedisSpider):
# ----3. 初始的 start_urls 改为 redis_key。
    redis_key = 'key'
# ----4. settings中 修改调度器类和去重类
    SCHEDULER = "scrapy_redis.scheduler.Scheduler"
    DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
    SCHEDULER_PERSIST = True
# ----5. 在settings.py文件中配置Redis
    REDIS_URL = redis://:[password]@host:port
         
 -----------------------------------------------------------------------
# ----1 导入分布式爬虫类
from scrapy_ redis.spiders import RedisSpider, RedisCrawlSpider
# ----2 继承分布式爬虫类
class BookSpider(RedisSpider):

# ----3 注销start_url & allowed_domains
# ----4 设置redis_key
    redis_key = 'key'
# ----5 设置__init__
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = list(filter(None, domain.split(',')))
        super(BookSpider, self).__init__(*args, **kwargs)
# ----5 修改配置文件
    SCHEDULER = "scrapy_redis.scheduler.Scheduler"
    DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
    SCHEDULER_PERSIST = True
    REDIS_URL = redis://:[password]@host:port
                
# 注：运行时要加上domain指定的域