基于scrapy中---全站爬取数据----CrawlSpider的使用

#数据源：糗事百科

爬虫代码：

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class QiubaiSpider(CrawlSpider):
    name = 'qiubai'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['https://www.qiushibaike.com/pic/']
    #第一页之后的所有url
    link=LinkExtractor(allow=r'/pic/page/\d+\?')
    #第一页的url
    link1=LinkExtractor(allow=r'/pic/$')
    rules = (
        Rule(link, callback='parse_item', follow=True),
        Rule(link1, callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        print(response.xpath('//*[@id="qiushi_tag_121352783"]/a/div/span/text()').extract_first())