scrapy 爬虫并把所有网址和所有图片对应起来写入到Excel中
items.py 数据条目
import scrapy
class DangdangItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
title=scrapy.Field()
link =scrapy.Field()
comment=scrapy.Field()
pic_addr=scrapy.Field()
dd.py 数据分析
# -*- coding: utf-8 -*-
import scrapy
import re
from dangdang.items import DangdangItem
from scrapy.http import Request
class DdSpider(scrapy.Spider):
name = 'dd'
allowed_domains = ['dangdang.com']
start_urls = ['http://category.dangdang.com/pg1-cid4003844.html']
def parse(self, response):
item=DangdangItem()
item['title']=response.xpath('//a[@name="itemlist-title"]/@title').extract()
item['link']=response.xpath('//a[@name="itemlist-title"]/@href').extract()
item['comment'] = response.xpath('//a[@name="itemlist-review"]/text()').extract()
item['pic_addr'] = response.xpath('//a/img/@data-original').extract()
lst=item['pic_addr']
lnk=item['link']
#相同图片地址整理到一起
i = 0
lst1 = []
while i < len(lst):
pat = 'http://.*?/.*?/.*?/([0-9]{5})'
data = re.compile(pat).findall(lst[i])
k = 0
j = 0
ll = []
while j < len(lst):
data1 = re.compile(pat).findall(lst[j])
if (data == data1):
ll.append(lst[j])
k += 1
if k > 1:
lst.pop(j)
j = j - 1
j += 1
lst1.append(ll)
i += 1
#网页链接和图片链接序号相匹配,为写入数据进行整理
lst = []
for m in range(0, len(lnk)):
pat1 = 'http://.*?/([0-9]{5})'
d = re.compile(pat1).findall(lnk[m])
for n in range(0, len(lst1)):
pat2 = 'http://.*?/.*?/.*?/([0-9]{5})'
d1 = re.compile(pat2).findall(lst1[n][0])
if d == d1:
lst.append(lst1[n])
break
item['pic_addr']=lst
yield item
for page in range(2,81):
url='http://category.dangdang.com/pg'+str(page)+'-cid4003844.html'
yield Request(url,callback=self.parse)
创建Excel表
import xlwt
book = xlwt.Workbook(encoding="utf-8")
sht = book.add_sheet("publisher", cell_overwrite_ok=True)
sht.write(0,0,u'序号')
sht.write(0,1,u'标题')
sht.write(0,2,u'链接')
sht.write(0,3,u'评论')
sht.write(0,4,u'图片链接1')
sht.write(0,5,u'图片链接2')
sht.write(0,6,u'图片链接3')
sht.write(0,7,u'图片链接4')
sht.write(0,8,u'图片链接5')
book.save("d:\\data\\dangdang\\dangdang.xls")
pipelines.py 数据写入
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import xlrd
from xlutils.copy import copy
class DangdangPipeline(object):
def process_item(self, item,spider):
rb = xlrd.open_workbook('d:\\data\\dangdang\\dangdang.xls')
sht = rb.sheets()[0]
nrows = sht.nrows
# ncols = sht.ncols+1
wb = copy(rb)
ws = wb.get_sheet(0)
for i in range(0,len(item['title'])):
title=item['title'][i]
link=item['link'][i]
comment=item['comment'][i]
pic_addr = item['pic_addr'][i]
try:
ws.write(nrows,0,nrows)
ws.write(nrows, 1, title)
ws.write(nrows, 2, link)
ws.write(nrows, 3, comment)
for j in range(0,len(pic_addr)):
ws.write(nrows,4+j,pic_addr[j])
nrows += 1
except Exception as err:
print(err)
wb.save('d:\\data\\dangdang\\dangdang.xls')
return item