Scrapy 爬取保险条款 -《狗嗨默示录》-
items.py
class IachinaItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() COMPANY = scrapy.Field() TYPE = scrapy.Field() PRODUCT = scrapy.Field() CLAUSE = scrapy.Field() CLAUSE_URL = scrapy.Field()
iachina.py
# -*- coding: utf-8 -*- import scrapy from IAChina.items import IachinaItem class IachinaSpider(scrapy.Spider): name = 'iachina' allowed_domains = ['old.iachina.cn'] start_urls = ['http://old.iachina.cn/product.php?action=company&ttype=2&page={}'.format(i) for i in range(1,4)] def parse(self, response): if not response: self.log("Company Page error -- %s"%response.url) for sel in response.xpath('//div[@class="prolist"]/ul/li/a'): item = IachinaItem() item['COMPANY'] = sel.xpath('text()').extract() company_href = sel.xpath('@href').extract_first() company_url = response.urljoin(company_href) yield scrapy.Request(url=company_url,meta={'item':item},callback=self.parse_type) def parse_type(self,response): if not response: self.log("Type Page erroe -- %s"%response.url) for sel in response.xpath('//div[@class="prolist"]/ul/li/a'): item = response.meta['item'] item['TYPE'] = sel.xpath('text()').extract() type_href = sel.xpath('@href').extract_first() type_url = response.urljoin(type_href) yield scrapy.Request(url=type_url, meta={'item': item}, callback=self.parse_product) def parse_product(self,response): if not response: self.log("Product Page erroe -- %s"%response.url) for sel in response.xpath('//div[@class="prolist"]/ul/li/a'): item = response.meta['item'] item['PRODUCT'] = sel.xpath('text()').extract() product_href = sel.xpath('@href').extract_first() product_url = response.urljoin(product_href) yield scrapy.Request(url=product_url, meta={'item': item}, callback=self.parse_clause) def parse_clause(self,response): if not response: self.log("Clause Page erroe -- %s"%response.url) for sel in response.xpath('//div[@class="prolist"]/table/tr[2]/td/a'): item = response.meta['item'] item['CLAUSE'] = sel.xpath('text()').extract() clause_href = sel.xpath('@href').extract_first() item['CLAUSE_URL'] = response.urljoin(clause_href) yield item