scrapy抓取校花网图片

一：基础版（抓取首页图片）

爬虫py文件代码：

# -*- coding: utf-8 -*-
import scrapy
import sys
import io
from scrapy.selector import Selector
from scrapy.http import Request
from ..items import Day96XiaohuaItem
import re
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')


class XiaohuaSpider(scrapy.Spider):
    name = 'xiaohua'
    allowed_domains = ['www.xueshengmai.com/hua/']
    start_urls = ['http://www.xueshengmai.com/hua/']

    def parse(self, response):
        # ------------持久化数据--------------
        hxs = Selector(response=response).xpath("//div[@class='item_t']/div[@class='img']/a/img").extract()
        # print(hxs)
        for i in hxs:
            # print(i)
            title = re.findall("alt=(.*) src=",i)[0].strip('"')+".jpg"
            src = "http://www.xueshengmai.com%s"%re.findall("src=(.*)>",i)[0].strip('"')
            print(title,src)
            item_obj = Day96XiaohuaItem(title=title, src=src)
            yield item_obj

items.py 代码：

import scrapy


class Day96XiaohuaItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title=scrapy.Field()
    src=scrapy.Field()

pipelines代码：

import requests

class Day96XiaohuaPipeline(object):
    def process_item(self, item, spider):
        file_path="imgs/%s"%item["title"]
        file_src=item["src"]
        f=open(file_path,"wb")
        img_date=requests.get(file_src)
        f.write(img_date.content)
        f.close()

二：分页抓取校花网图片

下面代码和上面如出一辙，只将不同的代码块写下：

# -*- coding: utf-8 -*-
import scrapy
import sys
import io
from scrapy.selector import Selector
from scrapy.http import Request
from ..items import Day96XiaohuaItem
import re

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')


class XiaohuaSpider(scrapy.Spider):
    name = 'xiaohua'
    allowed_domains = ['www.xueshengmai.com/hua/']
    start_urls = ['http://www.xueshengmai.com/hua/']

    def parse(self, response):
        # ------------持久化数据--------------
        hxs = Selector(response=response).xpath("//div[@class='item_t']/div[@class='img']/a/img").extract()
        # print(hxs)
        for i in hxs:
            # print(i)
            title = re.findall("alt=(.*) src=", i)[0].strip('"') + ".jpg"
            src = "http://www.xueshengmai.com%s" % re.findall("src=(.*)>", i)[0].strip('"')
            print("正在努力抓取校花网图片...",title, src)
            item_obj = Day96XiaohuaItem(title=title, src=src)
            yield item_obj

        hxs2 = Selector(response=response).xpath("//div[@class='page_num']/a").extract()
        for i in hxs2:
            page_number = re.findall(">(.*)<", i)[0]
            # print("=========",page_number)
            if page_number not in ["首页", "上一页", "下一页", "尾页"]:
                url = "http://www.xueshengmai.com/list-1-%s.html" % page_number
                # print("-------", url)
                # 将新要访问的url添加到调度器
                yield Request(url=url, callback=self.parse,dont_filter=True)

同时应该在settings中加上一句：

DEPTH_LIMIT = 1

否则程序将会一直深入下载，直到网站图片资源穷尽...