简易天猫爬虫
天猫商品数据爬取代码分享
虽然很简陋但是写这个程序我学到了一些新的技术,比如openpyxl库的使用,python的打包啊,设置图标啥的,还是收获很多.
闲话不多说,直接上代码
#导入需要的库
import re
import urllib.parse
import requests
from openpyxl import Workbook
from openpyxl.styles import Font,Alignment
import os
#创建工作簿对象
wb = Workbook()
#我们使用Workbook对象的默认创建的工作表
ws = wb.active
#标题命名
ws['A1'] = '商品名称'
ws['B1'] = '商品价格'
ws['C1'] = '产地'
ws['D1'] = '月成交量'
ws['E1'] = '商品链接'
#提示用户输入,查找的商品名,页数,保存的地址,和文件名
print('----欢迎使用----')
keyword = input("请输入你要查找的商品名称:")
frequency = int(input("请输入你要下载的页数(1~100):"))
name = input("请输入你要保存的文件名:")+'.xlsx'
#列宽判断的依据
width = 0
#冻结第一行
ws.freeze_panes = 'A2'
#标题格式设置居中
ws['A1'].alignment =Alignment(horizontal = 'center',vertical ='center')
ws['B1'].alignment =Alignment(horizontal = 'center',vertical ='center')
ws['C1'].alignment =Alignment(horizontal = 'center',vertical ='center')
ws['D1'].alignment =Alignment(horizontal = 'center',vertical ='center')
ws['E1'].alignment =Alignment(horizontal = 'center',vertical ='center')
#设置标题字号为20,加粗
ws['A1'].font = Font(size= 20,bold=True)
ws['B1'].font = Font(size= 20,bold=True)
ws['C1'].font = Font(size= 20,bold=True)
ws['D1'].font = Font(size= 20,bold=True)
ws['E1'].font = Font(size= 20,bold=True)
#设置商品价格,成交量,产地的列宽
ws.column_dimensions['B'].width = 20
ws.column_dimensions['C'].width = 20
ws.column_dimensions['D'].width = 20
#中文关键字的编码
keyword = urllib.parse.quote(keyword)
#请求头的处理
headers = {'user_agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.10 Safari/537.36',
'cookie':'cna=dzhnFJcvPFYCAcplZsR6KtPL; hng=CN%7Czh-CN%7CCNY%7C156; lid=su%E3%80%81%E9%9F%A9; otherx=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0; tk_trace=1; t=cf98821548a85be3261b5d3e02dfc50c; _tb_token_=53317e736e697; cookie2=171ec1fa788a97042140b1dc23ea8cbd; _m_h5_tk=ce5a4b969c5fee0ce43fbecb1c8b5698_1544366646609; _m_h5_tk_enc=08f79f8dd140168b3e95d192521140f3; x=__ll%3D-1%26_ato%3D0; whl=-1%260%260%260; uc1=cookie16=V32FPkk%2FxXMk5UvIbNtImtMfJQ%3D%3D&cookie21=UIHiLt3xTIkz&cookie15=UtASsssmOIJ0bQ%3D%3D&existShop=false&pas=0&cookie14=UoTYMh2PRIzYCw%3D%3D&tag=8&lng=zh_CN; uc3=vt3=F8dByR1fSHC9rqvO0Hw%3D&id2=UUGk2VnYR7N9og%3D%3D&nk2=saDewT4jhA05glO9&lg2=V32FPkk%2Fw0dUvg%3D%3D; tracknick=%5Cu4E00%5Cu5207%5Cu968F%5Cu7F18%5Cu4E28%5Cu4E36; _l_g_=Ug%3D%3D; ck1=""; unb=2967018108; lgc=%5Cu4E00%5Cu5207%5Cu968F%5Cu7F18%5Cu4E28%5Cu4E36; cookie1=AiVdFlFBrPvLkxJuDQ%2FIWWWqMYV30iZYcqUsqvmxAjc%3D; login=true; cookie17=UUGk2VnYR7N9og%3D%3D; _nk_=%5Cu4E00%5Cu5207%5Cu968F%5Cu7F18%5Cu4E28%5Cu4E36; uss=""; csg=da3a7af3; skt=22f8e8af802abead; enc=P0JAHDOULky9KTinsCWQ4Ib6YVG7q7qPW5KKCJd4YWKlwiYOGGRObgbMOWOpxn4w12VNH34hJK%2FVCxsPmDqs%2FQ%3D%3D; pnm_cku822=098%23E1hvc9vUvbpvUvCkvvvvvjiPR2FWljlUn2qw6jEUPmPZ1jrERFdO1jYUnLS9zjtUiQhvCvvvpZptvpvhvvCvpvGCvvpvvPMMvphvC9mvphvvvvyCvhQv7sg%2FjNpBKBh78BoxfXkXdiYso%2BLpjXe4Vc3Z0f06W3vOJ1kHsfUpeB6AxYjxRLwprj6OfwoKjd8rJm7g%2BfUz%2BsIIHYFpeiQa5javuphvmvvvpoX8LTuKkphvC9hvpyPw1byCvm9vvhCvvvvvvvvvBfIvvvjivvCVB9vv9LvvvhXVvvmCjvvvByOvvUhw; cq=ccp%3D0; swfstore=199766; isg=BG5uugvevlR2SM3ay3guf99Uv8ScezhtaUhC2pg36XEsew_VAP0keVBxNqcy_yqB'}
#url的处理
url1 = "https://list.tmall.com/search_product.htm?spm=a220m.1000858.0.0.36105702i4oQH9&s="
url2 ="&q=" +keyword+"&sort=s&style=g&from=..pc_1_searchbutton&active=2&type=pc#J_Filte"
#循环爬取每一页,正则提取商品链接,商品名,价格,销量
for i in range(1,frequency+1):
#跳过访问发生的异常,并输出异常信息
try:
print("----正在爬取第%d页----"%i)
url = url1 +str((i-1)*60)+url2
r = requests.get(url,headers = headers)
names = re.compile('target="_blank" title="(.*?)"',re.S).findall(r.text)
if(len(names)):
print('访问成功')
else:
print("访问失败,请更改代码的里cookie,或者明天再使用")
urls = re.compile('<div class="productImg-wrap">\n<a href="(.*?)" class="productImg" target="_blank" data-p="',re.S).findall(r.text)
chengjiaoliangs = re.compile('<span>月成交 <em>(.*?)笔',re.S).findall(r.text)
moneys = re.compile('<em title="(.*?)"><b>¥',re.S).findall(r.text)
#访问商品详细信息,爬取产地
wheres = []
for x in range(len(urls)):
#跳过获取商品地区信息时发生的异常,并输出异常
try:
wurl = 'http:'+urls[x]
w = requests.get(wurl,headers = headers)
wheres.append((re.compile('name="region" value="(.*?)"',re.S).findall(w.text))[0])
if(len(wheres[x])):
print('第%d页%d条商品信息爬取成功'%(i,x+1))
except Exception as er:
print(er)
if (len(names)):
print("----第%d页爬取成功----" % i)
print("----第%d页开始写入----"%i)
#循环写入excel表格
for y in range(1,len(urls)):
#跳过写入表格时发生的异常,并输出异常的信息
try:
#设置商品名称和商品链接自适应列宽
if(len(names[y-1])> width):
ws.column_dimensions['A'].width = 2*len(names[y-1])
if (len(urls[y - 1]) > width):
ws.column_dimensions['E'].width = len(urls[y - 1])
#分别写入数据
ws['A%d' % ((i - 1) * 60 + y + 1)] = names[y-1]
ws['B%d' % ((i - 1) * 60 + y + 1)] = moneys[y - 1]+'元'
ws['C%d' % ((i - 1) * 60 + y + 1)] = wheres[y - 1]
ws['D%d' % ((i - 1) * 60 + y + 1)] = chengjiaoliangs[y - 1]+'笔'
ws['E%d' % ((i - 1) * 60 + y + 1)] = 'http:'+ urls[y - 1]
print('----第%d页第%d条写入成功----'%(i,y+1))
except Exception as e:
print('----第%d页第%d条写入失败----'%(i,y+1))
print("----第%d页写入成功----" % i)
except Exception as err:
print(err)
#创建文件下载路径
path = './天猫数据爬取excel文件/'
if not os.path.exists(path):
os.mkdir(path)
#保存工作簿到指定路径
wb.save(path +name)
print('----已经全部写入----')
print('----感谢使用----')
os.system('pause')
