Nodejs爬取壁纸接口数据
- 最近在搞微信壁纸的一个小程序,里面的资源比较少,所以就想到用爬虫爬一些高清的一些壁纸,后来找到了一个比较不错的api接口可以直接使用的,下面我就直接用这个接口来爬了,不多说,直接上代码!
const download = require('download'); const axios = require('axios'); const fs = require('fs'); const _ = require('lodash') let headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36' }; function sleep(time) { return new Promise((reslove) => setTimeout(reslove, time)); } async function load(skip = 0) { const data = await axios .get('http://service.picasso.adesk.com/v1/vertical/category/4e4d610cdf714d2966000000/vertical', { headers, params: { limit: 30, // 每页固定返回30条 skip: skip, first: 0, order: 'hot' } }) .then((res) => { return res.data.res.vertical; }) .catch((err) => { console.log(err); }); const filePath = `${__dirname}/grid/grid.json`; await writeJson(data, filePath); await sleep(3000); if (skip < 1000) { load(skip + 30); } else { console.log('下载完成'); } await downloadFile(data) // 睡眠3秒防止网站403 await sleep(3000) if (skip < 1000) { load(skip + 30) } else { console.log('下载完成') } } // 这是我根据我个人的一些项目写成一个json文件方便导入我的云数据库中 async function writeJson(sourceImgs, filePath) { fs.readFile(filePath, 'utf-8', (err, data) => { if (err) throw err; const imgObj = JSON.parse(data); imgObj.time = new Date().getTime(); imgObj.user_id.$oid = ''; imgObj.category_id.$oid = ''; imgObj._id = ''; imgObj.image = data[0].wp; const tepmImgs = sourceImgs.map((item) => item.wp); imgObj.images = [ ...imgObj.images, ...tepmImgs ]; const str = JSON.stringify(imgObj, '', '\t'); fs.writeFile(filePath, str, (err) => { if (err) throw err; console.log('写入成功!'); }); }); } async function downloadFile(data) { for (let index = 0; index < data.length; index++) { const item = data[index]; // Path at which image will get downloaded const filePath = `${__dirname}/美女`; await download(item.wp, filePath, { filename: item.id + '.jpeg', headers }).then(() => { console.log(`Download ${item.id} Completed`); return; }); } } load();
// (1) 首先新建一个空文件夹,打开cmd,运行yarn init , 再安装一个依赖 yarn
// (2) 运行 当前js文件 node xxx.js -
大家可以使用微信扫一下小程序二维码体验效果!
-