使用scrapy爬取斗图网表情信息并加以分类存放 -《狗嗨默示录》-

 

demoSpider.py

# !/usr/bin/env python
# -*- coding: utf-8 -*-
import scrapy
from tutorial.items import DmozItem
import os
import requests


class DmozSpider(scrapy.Spider):
    name = "dmoz"
    allowed_domains = ["doutula.com"]
    start_urls = ["https://www.doutula.com/article/list/?page={}".format(i) for i in range(1,10)]
    count = 0

    def parse(self, response):
        for sel in response.xpath('//*[@id="home"]/div/div[1]'):
            item = DmozItem()
            item['link'] = sel.xpath('a/@href').extract()
            for i in range(len(item['link'])):
                yield scrapy.Request(item['link'][i], callback=self.parse_img)

    def parse_img(self, response):
        for sell in response.xpath('//div[@class="col-sm-9"]/li'):
            item = DmozItem()
            item['title'] = sell.xpath('div[@class="pic-title"]/h1/a/text()').extract()[0]
            item['img'] = sell.xpath('div[@class="pic-content"]//div[@class="artile_des"]/table/tbody/tr[1]/td/a/img/@alt').extract()
            item['url'] = sell.xpath('div[@class="pic-content"]/div[@class="artile_des"]/table/tbody/tr[1]/td/a/img/@src').extract()
            # print(item['title'])
            if not os.path.exists("D:\JetBrains\PyCharm 2017.2\PycharmProjects\Scrapy\\tutorial\\tutorial\spiders\down\{}".format(item['title'])):
                os.makedirs("D:\JetBrains\PyCharm 2017.2\PycharmProjects\Scrapy\\tutorial\\tutorial\spiders\down\{}".format(item['title']))
            else:
                print("存在重复文件夹 {}".format(item['title']))
            for i in range(len(item['img'])):
                filename = "down\{}".format(item['title'])+"\\{}".format(item['img'][i])+item['url'][i][-4:]
                print(filename,"==============================")
                if not os.path.exists(filename):
                    try:
                        img_req = requests.get(item['url'][i])
                    except requests.exceptions.MissingSchema:
                        new_url = 'http:'+item['url'][i]
                        img_req = requests.get(new_url)
                    with open(filename,'wb') as f:
                        f.write(img_req.content)
                else:
                    print("图片 [%s] 已存在!"%item['img'][i],"==========================")
                self.count += 1
        print("共计完成%s次下载"%self.count)