scrapy自动抓取蛋壳公寓最新房源信息并存入sql数据库
利用scrapy抓取蛋壳公寓上的房源信息，以北京市为例，目标url：https://www.dankegongyu.com/room/bj
思路分析
每次更新最新消息，都是在第一页上显示，因此考虑隔一段时间自动抓取第一页上的房源信息，实现抓取最新消息。
利用redis的set数据结构的特征，将每次抓取后的url存到redis中；
每次请求，将请求url与redis中的url对比，若redis中已存在该url，代表没有更新，忽略该次请求；若redis中不存在该url，代表该信息是新信息，抓取并将url存入到redis中。
分析页面源码，发现该网页属于静态网页；首先获取最新页面每条数据的url，请求该url，得到详细页面情况，所有数据均从详情页面获取。
代码实现
明确抓取字段
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy class DankeItem(scrapy.Item): """ 编辑带爬取信息字段 """ # 数据来源 source = scrapy.Field() # 抓取时间 utc_time = scrapy.Field() # 房间名称 room_name = scrapy.Field() # 房间租金 room_money = scrapy.Field() # 房间面积 room_area = scrapy.Field() # 房间编号 room_numb = scrapy.Field() # 房间户型 room_type = scrapy.Field() # 租房方式 rent_type = scrapy.Field() # 房间楼层 room_floor = scrapy.Field() # 所在区域 room_loca = scrapy.Field() # 所在楼盘 estate_name = scrapy.Field()
编写爬虫逻辑
# -*- coding: utf-8 -*- import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule from danke.items import DankeItem class DankeSpider(CrawlSpider): # 爬虫名 name = 'dkgy3' # 允许抓取的url allowed_domains = ['dankegongyu.com'] custom_settings = {'DOWNLOAD_DELAY': 0.2} # 请求开始的url start_urls = ['https://www.dankegongyu.com/room/sz'] # rules属性 rules = ( #编写匹配详情页的规则，抓取到详情页的链接后不用跟进 Rule(LinkExtractor(allow=r'https://www.dankegongyu.com/room/\d+'), callback='parse_detail', follow=False), ) def parse_detail(self, response): """ 解析详情页数据 :param response: :return: """ node_list = response.xpath('//div[@class="room-detail-right"]') for node in node_list: item = DankeItem() # 房间名称 room_name = node.xpath('./div/h1/text()') item['room_name'] = room_name.extract_first() # 房间租金 room_money = node.xpath('./div[@class="room-price"]/div/span').xpath('string(.)').extract_first() # 有的房子有首月租金，和普通租金不同，因此匹配方式也不同 if room_money: item['room_money'] = room_money else: room_money = node.xpath('./div[@class="room-price hot"]/div/div[@class="room-price-num"]/text()').extract_first() item['room_money'] = room_money print(room_money) # 房间面积 room_area = node.xpath('./*/div[@class="room-detail-box"]/div[1]/label/text()').extract_first().split('：')[-1] item['room_area'] = room_area # 房间编号 room_numb = node.xpath('./*/div[@class="room-detail-box"]/div[2]/label/text()').extract_first().split('：')[-1] item['room_numb'] = room_numb # 房间户型 room_type = node.xpath('./*/div[@class="room-detail-box"]/div[3]/label/text()').extract_first().split('：')[-1] item['room_type'] = room_type # 租房方式 rent_type = node.xpath('./*/div[@class="room-detail-box"]/div[3]/label/b/text()').extract_first().split('：')[ -1] item['rent_type'] = rent_type # 所在楼层 room_floor = node.xpath('./div[@class="room-list-box"]/div[2]/div[2]').xpath('string(.)').extract_first().split('：')[-1] item['room_floor'] = room_floor # 所在区域 room_loca = node.xpath('./div[@class="room-list-box"]/div[2]/div[3]/label/div/a[1]/text()').extract_first() item['room_loca'] = room_loca # 所在楼盘 estate_name = node.xpath('./div[@class="room-list-box"]/div[2]/div[3]/label/div/a[3]/text()').extract_first() item['estate_name'] = estate_name yield item
编写下载中间件
下载中间件中实现两个逻辑：添加随机请求头和url存入redis中
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html import time import random import hashlib import redis from scrapy.exceptions import IgnoreRequest from danke.settings import USER_AGENTS as ua class DankeSpiderMiddleware(object): def process_request(self, request, spider): """ 给每一个请求随机分配一个代理 :param request: :param spider: :return: """ user_agent = random.choice(ua) request.headers['User-Agent'] = user_agent class DankeRedisMiddleware(object): """ 将第一个页面上的每一个url放入redis的set类型中，防止重复爬取 """ # 连接redis def __init__(self): self.redis = redis.StrictRedis(host='39.106.116.21', port=6379, db=3) def process_request(self, request, spider): # 将来自详情页的链接存到redis中 if request.url.endswith(".html"): # MD5加密详情页链接 url_md5 = hashlib.md5(request.url.encode()).hexdigest() # 添加到redis，添加成功返回True,否则返回False result = self.redis.sadd('dk_url', url_md5) # 添加失败，说明链接已爬取，忽略该请求 if not result: raise IgnoreRequest
数据存储
# -*- coding: utf-8 -*- from datetime import datetime import pymysql class DankeSourcePipeline(object): def process_item(self, item, spider): item['source'] = spider.name item['utc_time'] = str(datetime.utcnow()) return item class DankePipeline(object): def __init__(self): self.conn = pymysql.connect( host='39.106.116.21', port=3306, database='***', user='***', password='****', charset='utf8' ) # 实例一个游标 self.cursor = self.conn.cursor() def process_item(self, item, spider): sql = ("insert into result_latest(标题, 租金, 面积, " "编号, 户型, 出租方式, 楼层, " "区域, 楼盘, 抓取时间, 数据来源)" "values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)") item = dict(item) data = [ item['room_name'], item['room_money'], item['room_area'], item['room_numb'], item['room_type'], item['rent_type'], item['room_floor'], item['room_loca'], item['estate_name'], item['utc_time'], item['source'], ] self.cursor.execute(sql, data) # 提交数据 self.conn.commit() return item def close_spider(self, spider): self.cursor.close() self.conn.close()
实现自动爬取
import os import time while True: """ 每隔20*60*60 自动爬取一次，实现自动更新 """ os.system("scrapy crawl dkgy3") time.sleep(20*60*60) # from scrapy import cmdline # cmdline.execute("scrapy crawl dkgy3".split())
完整代码
参见：https://github.com/zInPython/danke
