Scrapy: scrapy_redis

# 安装
pip3 install scrapy_redis
# 源码
https://github.com/rmax/scrapy-redis.git
# 文档
https://github.com/rmax/scrapy-redis

# 配置说明: https://github.com/rmax/scrapy-redis/wiki/Usage
REDIS_HOST = 'localhost'
REDIS_PORT = 6379
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 可暂停Spider
SCHEDULER_PERSIST = True
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
# SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderStack"

ITEM_PIPELINES = {
    'scrapy_redis.pipelines.RedisPipeline': 400,
}

# Spider
from scrapy_redis.spiders import RedisSpider

class BaiduSpider(RedisSpider):
    """Spider that reads urls from redis queue (myspider:start_urls)."""
    name = 'baiu'
    redis_key = 'myspider:baiu'
    # allowed_domains = ['baiu.com']

    def __init__(self, *args, **kwargs):
        # Dynamically define the allowed domains list.
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(BaiduSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        print(response.text)
        # return {
        #     'name': response.css('title::text').extract_first(),
        #     'url': response.url,
        # }

# CrawlSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_redis.spiders import RedisCrawlSpider


class FanqienovelSpider(RedisCrawlSpider):
    name = 'fanqienovel'
    redis_key = 'mycrawler:fanqienovel'
    # allowed_domains = ['baiu.com']

    rules = (
        # follow all links
        Rule(LinkExtractor(), callback='parse_page', follow=True),
    )

    def __init__(self, *args, **kwargs):
        # Dynamically define the allowed domains list.
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(FanqienovelSpider, self).__init__(*args, **kwargs)

    def parse_page(self, response):
        print(response.text)
        return {
            'name': response.css('title::text').extract_first(),
            'url': response.url,
        }