python3爬取动态网站图片
思路:
1、图片放在<image>XXX</image>标签中
2、利用fiddler抓包获取存放图片信息的js文件url
3、利用requests库获取html内容,然后获取其中图片id
4、利用fiddler抓取下载图片地址,结合图片id来下载图片(大文件)
# -*- coding:UTF-8 -*- import requests, json, time from contextlib import closing class get_photos(object): def __init__(self): self.photos_id = [] self.download_server = 'https://unsplash.com/photos/xxx/download?force=trues'#下载图片地址,通过fiddler抓包获取 self.target = 'https://unsplash.com/napi/photos?page=1&per_page=24'#存有图片信息的js文件地址 self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'} """ 函数说明:获取图片ID """ def get_ids(self): req = requests.get(url=self.target, headers=self.headers, verify=False) html = json.loads(req.text) print(type(html), len(html)) for i in range(len(html)): self.photos_id.append(html[i]['id'])#将列表html中获取id放到列表photos_id中 print(self.photos_id) time.sleep(1) """ 函数说明:图片下载 """ def download(self, photo_id, filename): self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'} self.target = self.download_server.replace('xxx', photo_id)#将xxx换成图片id #closing可以将任意对象转成上下文对象 with closing(requests.get(url=self.target, stream=True, verify = False, headers = self.headers)) as r: with open('%d.jpg' % filename, 'ab+') as f: #下载大文件,r.iter_content表示获取响应原始内容 for chunk in r.iter_content(chunk_size = 1024):#边下载边存硬盘,chunk_size表示以1024比例存数据 if chunk: f.write(chunk) f.flush()#强行把缓冲区中的内容放到磁盘中 if __name__ == '__main__': gp = get_photos()#类实例化 print('获取图片连接中:') gp.get_ids()#获取图片id print('图片下载中:') for i in range(len(gp.photos_id)): print('正在下载第%d张图片' % (i+1)) gp.download(gp.photos_id[i], (i+1))