1 # 安装
2 pip3 install scrapy_redis
3 # 源码
4 https://github.com/rmax/scrapy-redis.git
5 # 文档
6 https://github.com/rmax/scrapy-redis
7
8 # 配置说明: https://github.com/rmax/scrapy-redis/wiki/Usage
9 REDIS_HOST = 'localhost'
10 REDIS_PORT = 6379
11 DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
12 SCHEDULER = "scrapy_redis.scheduler.Scheduler"
13 # 可暂停Spider
14 SCHEDULER_PERSIST = True
15 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderPriorityQueue"
16 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderQueue"
17 # SCHEDULER_QUEUE_CLASS = "scrapy_redis.queue.SpiderStack"
18
19 ITEM_PIPELINES = {
20 'scrapy_redis.pipelines.RedisPipeline': 400,
21 }
22
23 # Spider
24 from scrapy_redis.spiders import RedisSpider
25
26 class BaiduSpider(RedisSpider):
27 """Spider that reads urls from redis queue (myspider:start_urls)."""
28 name = 'baiu'
29 redis_key = 'myspider:baiu'
30 # allowed_domains = ['baiu.com']
31
32 def __init__(self, *args, **kwargs):
33 # Dynamically define the allowed domains list.
34 domain = kwargs.pop('domain', '')
35 self.allowed_domains = filter(None, domain.split(','))
36 super(BaiduSpider, self).__init__(*args, **kwargs)
37
38 def parse(self, response):
39 print(response.text)
40 # return {
41 # 'name': response.css('title::text').extract_first(),
42 # 'url': response.url,
43 # }
44
45 # CrawlSpider
46 from scrapy.linkextractors import LinkExtractor
47 from scrapy.spiders import CrawlSpider, Rule
48 from scrapy_redis.spiders import RedisCrawlSpider
49
50
51 class FanqienovelSpider(RedisCrawlSpider):
52 name = 'fanqienovel'
53 redis_key = 'mycrawler:fanqienovel'
54 # allowed_domains = ['baiu.com']
55
56 rules = (
57 # follow all links
58 Rule(LinkExtractor(), callback='parse_page', follow=True),
59 )
60
61 def __init__(self, *args, **kwargs):
62 # Dynamically define the allowed domains list.
63 domain = kwargs.pop('domain', '')
64 self.allowed_domains = filter(None, domain.split(','))
65 super(FanqienovelSpider, self).__init__(*args, **kwargs)
66
67 def parse_page(self, response):
68 print(response.text)
69 return {
70 'name': response.css('title::text').extract_first(),
71 'url': response.url,
72 }
