简单实现一个初步的爬虫
Django: # 创建project django-admin startproject mysite cd mysite # 创建app python manage.py startapp app01 python manage.py startapp app02 # 启动项目 python manage.py runserver scrapy: # 创建project 项目名称 scrapy startproject xdb cd xdb #创建爬虫 爬虫名称 爬虫地址 scrapy genspider chouti chouti.com scrapy genspider cnblogs cnblogs.com # 启动爬虫 scrapy crawl chouti scrapy crawl chouti --nolog
""" 源码内容: 1. 判断当前XdbPipeline类中是否有from_crawler 有:obj = XdbPipeline.from_crawler(...) 否:obj = XdbPipeline() 2. obj.open_spider() 3. obj.process_item()|obj.process_item()|obj.process_item()| 4. obj.close_spider() """ from scrapy.exceptions import DropItem class XdbPipeline(object): def __init__(self, path): self.f = None self.path = path @classmethod def from_crawler(cls, crawler): ''' 初始化时候,用于创建pipeline对象 :param crawler: :return: ''' path = crawler.settings.get('HREF_FILE_PATH') return cls(path) def open_spider(self, spider): ''' 爬虫开始执行时,调用 :param spider: :return: ''' self.f = open(self.path, 'a+') def process_item(self, item, spider): # print(item.get("text")) self.f.write(item.get('href') + '\n') return item # 交给下一个Pipleline中的process_item方法去执行
return DropItem() # 后续的Pipeline中的process_item方法不再执行 def close_spider(self, spider): ''' 爬虫关闭时,被调用 :param spider: :return: ''' self.f.close()
持久化:pipelines pipelines.py class XdbPipeline(object): def __init__(self, path): self.f = None self.path = path @classmethod def from_crawler(cls, crawler): path = crawler.settings.get('HREF_FILE_PATH') return cls(path) def open_spider(self, spider): self.f = open(self.path, 'a+') def process_item(self, item, spider): # print(item.get("text")) self.f.write(item.get('href') + '\n') return item def close_spider(self, spider): self.f.close() settings.py ITEM_PIPELINES = { 'xdb.pipelines.XdbPipeline': 300, # 数字越小优先级越高, 范围0--1000 } items.py import scrapy class XdbItem(scrapy.Item): text = scrapy.Field() href = scrapy.Field() chouti.py import scrapy xdb.items import XdbItem class ChoutiSpider(scrapy.Spider): name = 'chouti' allowed_domains = ['chouti.com'] start_urls = ['http://chouti.com/'] def parse(self, response): content_list = response.xpath('//div[@class="link-con"]//div[@class="link-detail"]') for item in content_list: text = item.xpath('./a/text()').extract_first() href = item.xpath('./a/@href').extract_first() yield XdbItem(text=text, href=href)