python之scrapy篇(三)

一、创建工程(cmd)

scrapy startproject xxxx

二、编写item文件

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

from scrapy import Field, Item


class YouyuanItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # 用户名
    username = Field()
    # 年龄
    age = Field()
    # 头像图片链接
    herder_url = Field()
    # 相册图片
    image_url = Field()
    # 独白
    content = Field()
    # 籍贯
    place_from = Field()
    # 学历
    education = Field()
    # 兴趣爱好
    hobby = Field()
    # 个人主页
    source_url = Field()
    # 数据来源
    sourcec = Field()

三、编写settings文件

# -*- coding: utf-8 -*-

# Scrapy settings for youyuan project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'youyuan'

SPIDER_MODULES = ['youyuan.spiders']
NEWSPIDER_MODULE = 'youyuan.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'youyuan (+http://www.yourdomain.com)'

# Obey robots.txt rules
#ROBOTSTXT_OBEY = True

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#   'Accept-Language': 'en',
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'youyuan.middlewares.YouyuanSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    'youyuan.middlewares.YouyuanDownloaderMiddleware': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'youyuan.pipelines.YouyuanPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

四、进入spider文件创建自定义爬虫文件

scrapy genspider demo 'www.xxxx.com'

编写文件

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from youyuan.items import YouyuanItem
import re


class YySpider(CrawlSpider):
    name = 'yy'
    allowed_domains = ['xxxx.com']
    start_urls = ['http://www.xxxx.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p1/']

    # 匹配每一页链接匹配规则
    page_links = LinkExtractor(allow=(r"xxxx.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p\d+/"))
    # 每个主页的匹配规则
    profile_links = LinkExtractor(allow=(r"xxxxx.com/\d+-profile/"))

    rules = (
        Rule(page_links),
        Rule(profile_links, callback='parse_item'),
    )

    def parse_item(self, response):
        item = YouyuanItem()
        # 姓名
        item['username'] = self.get_username(response)
        # 年龄
        item['age'] = self.get_age(response)
        # 头像图片链接
        item['herder_url'] = self.get_herder_url(response)
        # 相册图片
        item['image_url'] = self.get_image_url(response)
        # 独白
        item['content'] = self.get_content(response)
        # 籍贯
        item['place_from'] = self.get_place_from(response)
        # 学历
        item['education'] = self.get_education(response)
        # 兴趣爱好
        item['hobby'] = self.get_hobby(response)
        # 个人主页
        item['source_url'] = response.url
        # 数据来源
        item['sourcec'] = "youyuan"

        yield item

    def get_username(self, response):
        username = response.xpath("//dl[@class='personal_cen']//div[@class='main']/strong/text()").extract()
        if len(username):
            username = username[0]
        else:
            username = "NULL"
        return username.split()

    def get_age(self, response):
        age = response.xpath("//dl[@class='personal_cen']//dd/p/text()").extract()
        if len(age):
            age = re.findall(u"\d+岁", age[0])[0]
        else:
            age = "NULL"
        return age.strip()

    def get_herder_url(self, response):
        herder_url = response.xpath("//dl[@class='personal_cen']//dt/img/@src").extract()
        if len(herder_url):
            herder_url = herder_url[0]
        else:
            herder_url = "NULL"
        return herder_url.strip()

    def get_image_url(self, response):
        image_url = response.xpath("//div[@class='ph_show']/ul/li/a/img/@src").extract()
        if len(image_url):
            image_url = image_url
        else:
            image_url = "NULL"
        return image_url

    def get_content(self, response):
        content = response.xpath("//div[@class='pre_data']/ul/li/p/text()").extract()
        if len(content):
            content = content[0]
        else:
            content = "NULL"
        return content.strip()

    def get_place_from(self, response):
        place_from = response.xpath("//div[@class='pre_data']/ul/li[2]//ol[1]/li[1]/span/text()").extract()
        if len(place_from):
            place_from = place_from[0]
        else:
            place_from = "NULL"
        return place_from.strip()

    def get_education(self, response):
        education = response.xpath("//div[@class='pre_data']/ul/li[3]//ol[2]/li[2]/span/text()").extract()
        if len(education):
            education = education[0]
        else:
            education = "NULL"
        return education.strip()

    def get_hobby(self, response):
        hobby = response.xpath("//dl[@class='personal_cen']//ol/li/text()").extract()
        if len(hobby):
            hobby = ",".join(hobby).replace(" ", "")
        else:
            hobby = "NULL"
        return hobby.strip()

五、运行

scrapy crawl xxxx

OVER！