node-fetch Advanced Usage All In One

node-fetch Advanced Usage All In One

fetch

// stream

https://www.npmjs.com/package/node-fetch#streams

demos

Node.js web crawler

import fetch from "node-fetch";

import path from 'node:path';
import {fileURLToPath} from 'node:url';

// import fs from 'node:fs';
import {createWriteStream} from 'node:fs';
import {pipeline} from 'node:stream';
import {promisify} from 'node:util'

// const __filename = fileURLToPath(import.meta.url);
// const __dirname = path.dirname(__filename);
// console.log(`import.meta.url`, import.meta.url)
// console.log(`__dirname`, __dirname)


async function downloadFile(url, path) {
  const streamPipeline = promisify(pipeline);
  fetch(url).then(async (res) => {
    if (!res.ok) {
      throw new Error(`unexpected response ${res.statusText}`);
    }
    console.log(`✅ res =`, res)
    return await streamPipeline(res.body, createWriteStream(path));
  }).catch(err => {
    console.log(`❌ err =`, err)
  }).finally(() => {
    console.log(`finally 👻`)
  })
}

// async function downloadFile(url, path) {
//   const streamPipeline = promisify(pipeline);
//   const res = await fetch(url)
//   if (!res.ok) {
//     throw new Error(`unexpected response ${res.statusText}`);
//   }
//   // console.log(`✅ res =`, res)
//   await streamPipeline(res.body, createWriteStream(path));
// }


// const url = `https://edu-vod.lagou.com/sv/2daa3bb9-1765150ee6e/2daa3bb9-1765150ee6e.mp4`
// const url = `https://cdn.xgqfrms.xyz/video/web-testing.mp4`
const url = `https://cdn.xgqfrms.xyz/video/web-testing.mp5`
await downloadFile(url, "./test.mp4");

/*

$ node ./node-fetch.js

*/

axios

import axios from 'axios';
// import axios, {isCancel, AxiosError} from 'axios';

// import fs from 'node:fs';
import {createWriteStream} from 'node:fs';

// import path from 'path';
// import { fileURLToPath } from 'url';

// const __filename = fileURLToPath(import.meta.url);
// const __dirname = path.dirname(__filename);
// console.log(`import.meta.url`, import.meta.url)
// console.log(`__dirname`, __dirname)

// async function downloadFile(url, path) {
//   const res = await axios({
//     url,
//     method: "GET",
//     responseType: "stream",
//   });
//   console.log(`✅ content-type =`, res.headers['content-type'])
//   res.data.pipe(createWriteStream(path));
// }

async function downloadFile(url, path) {
  await axios({
    url,
    method: "GET",
    responseType: "stream",
  }).then(res => {
    console.log(`✅ content-type =`, res.headers['content-type'])
    return res.data.pipe(createWriteStream(path));
  }).catch(err => {
    console.log(`❌ err =`, err)
  }).finally(() => {
    console.log(`finally 👻`)
  })
}


// const url = `https://edu-vod.lagou.com/sv/2daa3bb9-1765150ee6e/2daa3bb9-1765150ee6e.mp4`
const url = `https://cdn.xgqfrms.xyz/video/web-testing.mp4`
await downloadFile(url, "./test.mp4");


/*

$ node ./axios.js

*/

(🐞 反爬虫测试！打击盗版⚠️）如果你看到这个信息, 说明这是一篇剽窃的文章，请访问 https://www.cnblogs.com/xgqfrms/ 查看原创文章！

Streams

Types of streams

There are four fundamental stream types within Node.js:

1. Writable: streams to which data can be written (for example, fs.createWriteStream()).
2. Readable: streams from which data can be read (for example, fs.createReadStream()).
3. Duplex: streams that are both Readable and Writable (for example, net.Socket).
4. Transform: Duplex streams that can modify or transform the data as it is written and read (for example, zlib.createDeflate()).

Additionally, this module includes the utility functions stream.pipeline(), stream.finished(), stream.Readable.from() and stream.addAbortSignal().

import { pipeline } from 'node:stream/promises';
import { createReadStream, createWriteStream } from 'node:fs';
import { createGzip } from 'node:zlib';

await pipeline(
  createReadStream('archive.tar'),
  createGzip(),
  createWriteStream('archive.tar.gz'),
);

console.log('Pipeline succeeded.');

https://nodejs.org/dist/latest-v18.x/docs/api/stream.html

The WHATWG Streams Standard (or "web streams") defines an API for handling streaming data.
It is similar to the Node.js Streams API but emerged later and has become the "standard" API for streaming data across many JavaScript environments.

There are three primary types of objects:

1. ReadableStream - Represents a source of streaming data.
2. WritableStream - Represents a destination for streaming data.
3. TransformStream - Represents an algorithm for transforming streaming data.

import {
  ReadableStream,
} from 'node:stream/web';

import {
  setInterval as every,
} from 'node:timers/promises';

import {
  performance,
} from 'node:perf_hooks';

const SECOND = 1000;

const stream = new ReadableStream({
  async start(controller) {
    for await (const _ of every(SECOND))
      controller.enqueue(performance.now());
  },
});

for await (const value of stream)
  console.log(value);

https://nodejs.org/dist/latest-v18.x/docs/api/webstreams.html

Modules: node:module API

ESM

// module.mjs
// In an ECMAScript module
import { builtinModules as builtin } from 'node:module';

CJS

// module.cjs
// In a CommonJS module
const builtin = require('node:module').builtinModules;

https://nodejs.org/dist/latest-v18.x/docs/api/module.html

node:fs vs fs

import fs from 'node:fs';

import fs from 'fs';

node: 类似 https://, file:// 是一种通信协议 ❓

https://stackoverflow.com/questions/67554506/what-are-the-nodefs-nodepath-etc-modules

refs

https://byby.dev/node-download-image

---

---

©xgqfrms 2012-2021

www.cnblogs.com/xgqfrms 发布文章使用：只允许注册用户才可以访问！

原创文章，版权所有©️xgqfrms, 禁止转载 🈲️，侵权必究⚠️！

---