scrapy 爬取图片 最基本操作

使用scrapy里自带的Image功能下载,下面贴代码，解释在代码的注释里。

 

items.py

import scrapy

class ImageItem(scrapy.Item):
    #注意这里的item是ImageItem

    image_urls = scrapy.Field()
    images = scrapy.Field()

#image_urls和images是固定的,不能改名字

 

settings.py

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'
#上面只是个访问header，加个降低被拒绝的保险

ITEM_PIPELINES = {
    'scrapy.pipelines.images.ImagesPipeline': 1
}
#打开Images的通道

IMAGES_STORE = 'F:\\pics'
#一定要设置这个存储地址，要是真实的硬盘，可以不创建pics文件夹，会自己生成，还生成默认的full文件夹

 

 spider.py(这里是carhome） 

import scrapy

from car.items import ImageItem

class CarhomeSpider(scrapy.Spider):
    name = 'carhome'
    allowed_domains = ['sohu.com']
    start_urls = ['http://www.sohu.com/a/337634404_100032610']
    download_delay = 1

    def parse(self, response):
        item = ImageItem()
        srcs = response.css('.article img::attr(src)').extract() #用css方法找到的所有图片地址
        item['image_urls'] = srcs
        yield item

 

pipelines.py 

from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
from scrapy.http import Request
#这里的两个函数get_media_requests和item_completed都是scrapy的内置函数，想重命名的就这这里操作
#可以直接复制这里的代码就可以用了
class MyImagesPipeline(ImagesPipeline) :
    def get_media_requests(self, item, info) :
        for image_url in item['image_urls'] :
            yield Request(image_url)

    def item_completed(self, results, item, info) :
        image_path = [x['path'] for ok, x in results if ok]
        if not image_path :
            raise DropItem('Item contains no images')
        item['image_paths'] = image_path
        return item

 代码就这么多，下面执行一下：

 

 文件夹结果：

 

 还不会的快去试试吧！