Node.js爬取百度图片瀑布流，使用class类封装。

//爬取百度高清图片
const phantom = require('phantom')
const express = require('express');
const app = express();
const fs= require('fs');
const cheerio = require('cheerio');
const request = require('request')
let server = app.listen(2000, function () {
    let host = server.address().address;
    let port = server.address().port;
    console.log('Your App is running at http://%s:%s', host, port);
});

class stealData {

    constructor() {
        // this.base_url = 'https://image.baidu.com/search/index?tn=baiduimage&ct=201326592&lm=-1&cl=2&ie=gb18030&word=%B6%AF%C2%FE%B1%DA%D6%BD&fr=ala&ala=1&pos=0&alatpl=wallpaper&oriquery=%E5%8A%A8%E6%BC%AB%E5%A3%81%E7%BA%B8'; //要爬取的网站
        // this.base_url = 'https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&hs=0&xthttps=111111&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=%E5%8A%A8%E6%BC%AB+%E5%A3%81%E7%BA%B8&oq=%E5%8A%A8%E6%BC%AB+%E5%A3%81%E7%BA%B8&rsp=-1'
        this.base_url = 'https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=index&fr=&hs=0&xthttps=111111&sf=1&fmq=&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word=%E5%8A%A8%E6%BC%AB+%E5%A3%81%E7%BA%B8&oq=%E5%8A%A8%E6%BC%AB+%E5%A3%81%E7%BA%B8&rsp=-1'

        this.current_page = 1;
        this.result_list = [];
        this.a=''
    }
    async init() {
        const instance = await phantom.create();//创建一个实例
        try {
            await this.openNet()//打开网页
            await this.getLoadPictures();//获取高清图片地址
            await this.imgSave(0);//下载图片
            await instance.exit()//图片下完之后退出phantomjs环境
        } catch (e) {
            console.log(e);
        }
    }

    sleep(time) {
        return new Promise((resolve) => {
            console.log(`自动睡眠中，${time / 1000}秒后重新发送请求......`)
            setTimeout(() => {
                resolve();
            }, time);
        });
    }//请求延迟时间，防止ip被封
    delay(second) {
        return new Promise((resolve) => {
            setTimeout(resolve, second * 1000);
        });
    }//延迟时间
    async openNet(){
        await this.pageScroll(0)
    }

    async pageScroll(i) {
        const instance = await phantom.create();//创建一个实例
        const page = await instance.createPage();
        const that = this
        const status = await page.open(this.base_url);//打开网站，返回的是一个状态
        //网页加载的初始浏览器窗口大小
        const size = await page.property('viewportSize', {
            width: 1920,
            height: 1080
        })
        await that.delay(5)
        //滚动浏览器
        await page.property('scrollPosition', {
            left: 0,
            top: 1000 * i
        })

        let content = await page.property('content')//获取dom元素
        let $ = cheerio.load(content)
        console.log($('.imgbox').length)
        //如果图片数量少于20个 就一直往下滚
        if($('.imgbox').length < 20) {
            await this.pageScroll(++i)//回调自己 知道满足条件。也可使用for循环 可能速度有点慢
        }
        //所有的存储图片的dom元素都放在这里面
        this.a = $('.imgitem')
    }
    //获取到缩略图集合
    async getLoadPictures(a) {
        const result_list = [];
        let instance = await phantom.create();
        let page = await instance.createPage();
        let content = await page.property('content')
        // let status = await page.open(this.base_url)
        let $ = cheerio.load(content)
        const that = this
        //把所有的高清大图链接都统计起来
        this.a.each((index, element) => {
            result_list.push({
                // title: $(element).find('.imgbox a').text(),
                down_loda_url: ('https://image.baidu.com'+$(element).find('a').attr('href'))
            });
        });
        that.result_list.push(...result_list);
        // console.log('that.result_list',that.result_list[0])
    }
    //取到高清图链接 下载
    async  imgSave(i) {
        let instance = await phantom.create();
        let page = await instance.createPage();
        let status = await page.open(this.result_list[i].down_loda_url)//打开高清大图链接
        await this.delay(2)
        let content = await page.property('content')
        let $ = cheerio.load(content)
        let src = $('#currentImg').attr('src')//获取高清大图的链接
        console.log('statue',status)
        console.log('src',src)
        this.save(src,i)//保存图片
        if(i<this.result_list.length) {
            await this.imgSave(++i)//回掉自己 保存下一张图片。也可通过for循环 取到所有大图的链接 再集中下载 可能速度有点慢

        }
    }
    //保存图片函数
    save(url,i) {
       console.log(`开始写入第${i+1}张`)
        let ext = url.split('.').pop()
        request(url).pipe(fs.createWriteStream(`./image/${new Date().getTime()}.${ext}`));
        console.log(`写入成功`)
    }


}

const thief = new stealData('xxx_url');
thief.init();

做了一些优化

 

//爬取百度高清图片
const phantom = require('phantom')
const express = require('express');
const app = express();
const fs= require('fs');
const cheerio = require('cheerio');
const request = require('request')
let server = app.listen(2000, function () {
    let host = server.address().address;
    let port = server.address().port;
    console.log('Your App is running at http://%s:%s', host, port);
});

class stealData {

    constructor() {
        this.base_url = 'https://image.baidu.com/search/index?ct=201326592&z=&tn=baiduimage&word=%E6%BC%AB%E5%A8%81%E5%9B%BE%E7%89%87&pn=0&ie=utf-8&oe=utf-8&cl=2&lm=-1&fr=ala&se=&sme=&width=1920&height=1080'
        this.current_page = 1;
        this.result_list = [];
        this.a='';
        this.urllist = []
    }
    async init() {
        const instance = await phantom.create();//创建一个实例
        try {
            await this.openNet()//打开网页
            await this.getLoadPictures();//获取缩略图图片地址
            await this.getrealPictures();//获取高清图片地址并下载
        } catch (e) {
            console.log(e);
        }
    }

    sleep(time) {
        return new Promise((resolve) => {
            console.log(`自动睡眠中，${time / 1000}秒后重新发送请求......`)
            setTimeout(() => {
                resolve();
            }, time);
        });
    }//请求延迟时间，防止ip被封
    delay(second) {
        return new Promise((resolve) => {
            setTimeout(resolve, second * 1000);
        });
    }//延迟时间
    async openNet(){
        await this.pageScroll(0)
    }

    async pageScroll(i) {
        const instance = await phantom.create();//创建一个实例
        const page = await instance.createPage();
        const that = this
        const status = await page.open(this.base_url);//打开网站，返回的是一个状态
        //网页加载的初始浏览器窗口大小
        const size = await page.property('viewportSize', {
            width: 1920,
            height: 1080
        })
        await that.delay(5)
        //滚动浏览器
        await page.property('scrollPosition', {
            left: 0,
            top: 1000 * i
        })

        let content = await page.property('content')//获取dom元素
        let $ = cheerio.load(content)
        console.log($('.imgbox').length)
        //如果图片数量少于20个 就一直往下滚
        if($('.imgbox').length < 20) {
            await this.pageScroll(++i)//回调自己 知道满足条件。也可使用for循环 可能速度有点慢
        }
        //所有的存储图片的dom元素都放在这里面
        this.a = $('.imgitem')
    }
    //获取到缩略图集合
    async getLoadPictures(a) {
        const result_list = [];
        let instance = await phantom.create();
        let page = await instance.createPage();
        let content = await page.property('content')
        let $ = cheerio.load(content)
        const that = this
        //把所有的缩略图链接都统计起来
        this.a.each((index, element) => {
            result_list.push({
                // title: $(element).find('.imgbox a').text(),
                down_loda_url: ('https://image.baidu.com'+$(element).find('a').attr('href'))
            });
        });
        that.result_list.push(...result_list);
    }
    //取到每一个缩略图对应高清图链接并下载
    async  getrealPictures() {
        let instance = await phantom.create();
        let page = await instance.createPage();
        for(let i=0;i<this.result_list.length;i++){
            try {
                let content = await page.property('content')
                let status = await page.open(this.result_list[i].down_loda_url)
                await this.delay(2)
                let $ = cheerio.load(content)
                let src = $('#currentImg').attr('src')//获取高清大图的链接
                let ext = src.split('.').pop()
                console.log('src',src)
                console.log(`开始写入第${i+1}张`)
                await request(src).pipe(fs.createWriteStream(`./Marvel/${new Date().getTime()}.${ext}`));
                await this.sleep(3000)//防止被封
                console.log(`写入成功`)
                 if(i==this.result_list.length-1){
                    console.log('跳出下载')
                    instance.exit()
                }
            }catch (e) {
                console.log('errorheyu:',e)
            }

        }


    }

}

const thief = new stealData('xxx_url');
thief.init();

 

 

 

 

 

参考：1、分分钟教你用node写个爬虫

　　　2、Node.js爬虫实战，爬去图片到本地

　　　3、爬取瀑布流网页高清图

　　　4、PhantomJs的用法

　　　5、NodeJs优秀工具之——nightmare

　　　6、腾讯云上Phantom用例