nodejs抓取接口数据（axios）

 

demo：

var axios = require("axios");
var fs = require("fs");
var nodeschedule = require("node-schedule");

var hotListUrl = "https://weibo.com/ajax/side/hotSearch";

var option = {
    headers: {
        'authority': 'weibo.com',
        'pragma': 'no-cache',
        'cache-control': 'no-cache',
        'server-version': 'v2022.09.19.2',
        'x-xsrf-token': 'HZonfoACpKXZIMOMIOMc8y0b',
        'traceparent': '00-d925421b95d78ed73bafd2caed0cbd85-d21a698eff02f887-00',
        'sec-ch-ua-mobile': '?0',
        'client-version': 'v2.35.2',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.51 Safari/537.36',
        'accept': 'application/json, text/plain, */*',
        'x-requested-with': 'XMLHttpRequest',
        'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"',
        'sec-ch-ua-platform': '"Windows"',
        'sec-fetch-site': 'same-origin',
        'sec-fetch-mode': 'cors',
        'sec-fetch-dest': 'empty',
        'referer': 'https://weibo.com/newlogin?url=https%3A%2F%2Fweibo.com%2F',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,zh-TW;q=0.7',
        'cookie': 'SINAGLOBAL=5037293883765.11.1592755698570; SCF=AvGPjvoIgatl_Km54N5lsBGA06flluaOTUcjmG2i2L2YeK4HOTa1Voar-0Ev3FcDOINoYW122S9gO8896pNNJ90.; ALF=1669559693; SUB=_2AkMVQ5FKf8NxqwJRmPATymviaIp0zgvEieKjH2CRJRMxHRl-yT9jqnwDtRB6PsO_WLgUxUbfBmN0a8DQ-WByGIgx997S; SUBP=0033WrSXqPxfM72-Ws9jqgMF55529P9D9WFCwjKpB5Xfbk0rIDJhSqQu; UOR=,,www.baidu.com; ULV=1646206596535:8:1:1:682239284045.305.1646206596530:1632125497107; XSRF-TOKEN=HZonfoACpKXZIMOMIOMc8y0b; WBPSESS=durPiJxsbzq5XDaI2wW0N6ET-b2ytp8n58jgCm5B6U2fKwwMxWGcDGWqRlDvg85bX24BypCou0b2NQOLN37NXJngjuq3QEQ_OnB1GSws99oIKpNDkTbFEyDoVbHitmUguOtnqxtlUi3Lz2aC7DZqH-nDFDpiBi4P89n0g-xepkQ=',
    }

}

function parseData(data) {
    var result = []
    realtime = data.data.realtime
    //对于数组对象要这么循环取值
    realtime.forEach(function (element) {
        var title = element.word
        var num = element.num
        result.push({
            title,
            num
        })

    })
    return result

}

function getHotList() {
    return new Promise(((resolve, reject) => {
        axios.get(hotListUrl, option).then(function (resposne) {
            // console.log(resposne.data)
            var result = parseData(resposne.data)
            console.log(result)
            result.length ? resolve(result) : reject("err")
        }).catch(function (err) {
            console.log(err)
        }).finally(function () {
            console.log("完成一次抓取")

        })

    }))


}

/*
 * schedule
*    *    *    *    *    *
┬    ┬    ┬    ┬    ┬    ┬
│    │    │    │    │    │
│    │    │    │    │    └ day of week (0 - 7) (0 or 7 is Sun)
│    │    │    │    └───── month (1 - 12)
│    │    │    └────────── day of month (1 - 31)
│    │    └─────────────── hour (0 - 23)
│    └──────────────────── minute (0 - 59)
└───────────────────────── second (0 - 59, OPTIONAL)
 */
/**
 * 每分钟第30秒定时执行爬取任务
 */

nodeschedule.scheduleJob("*/10 * * * * *", async function () {

    try {
        var d1 = new Date();
        console.log(d1.getTime());
        var res = await getHotList();
        console.log(res)
        fs.open("result.json", "a", function (err1, fd) {
            if (!err1) {
                fs.write(fd, JSON.stringify(res), function (err2) {
                    if (!err2) {
                        fs.close(fd, function (err3) {
                            if (!err3) {
                                console.log("关闭文件成功")
                            } else {
                                console.log("关闭文件失败")
                            }

                        })


                    } else {
                        console.log("写入文件失败")
                    }

                })


            } else {
                console.log("打开文件失败")
            }

        })

    } catch (err) {
        console.log(err)
    }
})