[Node.js] Fetch csv data and parse

We use needleas a client on Node.js express server for fetching the data or site. https://www.npmjs.com/package/needle

The leanest and most handsome HTTP client in the Nodelands.

var needle = require('needle');

needle.get('http://www.google.com', function(error, response) {
  if (!error && response.statusCode == 200)
    console.log(response.body);
});
Callbacks not floating your boat? Needle got your back.

var data = {
  file: '/home/johnlennon/walrus.png',
  content_type: 'image/png'
};

// the callback is optional, and needle returns a `readableStream` object
// that triggers a 'done' event when the request/response process is complete.
needle
  .post('https://my.server.com/foo', data, { multipart: true })
  .on('readable', function() { /* eat your chunks */ })
  .on('done', function(err) {
    console.log('Ready-o!');
  })

 

Then we use csv-parse to parse the content: https://www.npmjs.com/package/csv-parse

import assert from 'assert';
import { parse } from 'csv-parse';

const records = [];
// Initialize the parser
const parser = parse({
  delimiter: ':'
});
// Use the readable stream api to consume records
parser.on('readable', function(){
  let record;
  while ((record = parser.read()) !== null) {
    records.push(record);
  }
});
// Catch any error
parser.on('error', function(err){
  console.error(err.message);
});
// Test that the parsed records matched the expected records
parser.on('end', function(){
  assert.deepStrictEqual(
    records,
    [
      [ 'root','x','0','0','root','/root','/bin/bash' ],
      [ 'someone','x','1022','1022','','/home/someone','/bin/bash' ]
    ]
  );
});
// Write data to the stream
parser.write("root:x:0:0:root:/root:/bin/bash\n");
parser.write("someone:x:1022:1022::/home/someone:/bin/bash\n");
// Close the readable stream
parser.end();

 

---- 

In this example, the pipe function is used to stream the data from the remote CSV file to a local file, and the on method is used to listen for the done event, which is emitted once the file has finished downloading. The contents of the local file are then read using fs.readFileSync and parsed using csv-parse.

const needle = require('needle');
const parse = require('csv-parse');
const fs = require('fs');

needle.get('https://example.com/file.csv')
  .pipe(fs.createWriteStream('file.csv'))
  .on('done', function() {
    // read the file once it's finished downloading
    const csv = fs.readFileSync('file.csv', 'utf-8');
    // parse the CSV content
    const records = parse(csv, {
      columns: true,
      trim: true
    });
    // do something with the parsed data
    console.log(records);
  });