scrapy抓取校花网图片
一：基础版（抓取首页图片）
爬虫py文件代码：
1 # -*- coding: utf-8 -*- 2 import scrapy 3 import sys 4 import io 5 from scrapy.selector import Selector 6 from scrapy.http import Request 7 from ..items import Day96XiaohuaItem 8 import re 9 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') 10 11 12 class XiaohuaSpider(scrapy.Spider): 13 name = 'xiaohua' 14 allowed_domains = ['www.xueshengmai.com/hua/'] 15 start_urls = ['http://www.xueshengmai.com/hua/'] 16 17 def parse(self, response): 18 # ------------持久化数据-------------- 19 hxs = Selector(response=response).xpath("//div[@class='item_t']/div[@class='img']/a/img").extract() 20 # print(hxs) 21 for i in hxs: 22 # print(i) 23 title = re.findall("alt=(.*) src=",i)[0].strip('"')+".jpg" 24 src = "http://www.xueshengmai.com%s"%re.findall("src=(.*)>",i)[0].strip('"') 25 print(title,src) 26 item_obj = Day96XiaohuaItem(title=title, src=src) 27 yield item_obj
items.py 代码：
1 import scrapy 2 3 4 class Day96XiaohuaItem(scrapy.Item): 5 # define the fields for your item here like: 6 # name = scrapy.Field() 7 title=scrapy.Field() 8 src=scrapy.Field()
pipelines代码：
import requests class Day96XiaohuaPipeline(object): def process_item(self, item, spider): file_path="imgs/%s"%item["title"] file_src=item["src"] f=open(file_path,"wb") img_date=requests.get(file_src) f.write(img_date.content) f.close()
二：分页抓取校花网图片
下面代码和上面如出一辙，只将不同的代码块写下：
1 # -*- coding: utf-8 -*- 2 import scrapy 3 import sys 4 import io 5 from scrapy.selector import Selector 6 from scrapy.http import Request 7 from ..items import Day96XiaohuaItem 8 import re 9 10 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') 11 12 13 class XiaohuaSpider(scrapy.Spider): 14 name = 'xiaohua' 15 allowed_domains = ['www.xueshengmai.com/hua/'] 16 start_urls = ['http://www.xueshengmai.com/hua/'] 17 18 def parse(self, response): 19 # ------------持久化数据-------------- 20 hxs = Selector(response=response).xpath("//div[@class='item_t']/div[@class='img']/a/img").extract() 21 # print(hxs) 22 for i in hxs: 23 # print(i) 24 title = re.findall("alt=(.*) src=", i)[0].strip('"') + ".jpg" 25 src = "http://www.xueshengmai.com%s" % re.findall("src=(.*)>", i)[0].strip('"') 26 print("正在努力抓取校花网图片...",title, src) 27 item_obj = Day96XiaohuaItem(title=title, src=src) 28 yield item_obj 29 30 hxs2 = Selector(response=response).xpath("//div[@class='page_num']/a").extract() 31 for i in hxs2: 32 page_number = re.findall(">(.*)<", i)[0] 33 # print("=========",page_number) 34 if page_number not in ["首页", "上一页", "下一页", "尾页"]: 35 url = "http://www.xueshengmai.com/list-1-%s.html" % page_number 36 # print("-------", url) 37 # 将新要访问的url添加到调度器 38 yield Request(url=url, callback=self.parse,dont_filter=True)
同时应该在settings中加上一句：
1 DEPTH_LIMIT = 1
否则程序将会一直深入下载，直到网站图片资源穷尽...
