How do I run multiple node streams in sequence?

Question

Given two streams, stream1, stream2, how can I run them in sequence, throwing if any fails?

I'm looking for something simpler than this:

stream1.on('end',function(){
   stream2.on('end',done);
});
stream1.on('error',function(error){done(error);});
stream2.on('error',function(error){done(error);});

Thanks.

Could you give more details what's your stream use case? How do you process stream data? This could lead to better answers — saintedlama, Jan 27 '16 at 14:41
It's really just a general question. I saw a blog post that had a bad-looking pyramid of streams and couldn't find a simple solution via Google. — Francisc, Jan 28 '16 at 13:35

score 2 · Accepted Answer · answered Jan 27 '16 at 13:59

2

there are some ways to do that, check next link, it gonna help to how to write some code in node in a elegant way:

Node.js FLOW

Hope it what you need.

answered Jan 27 '16 at 13:59

SAMUEL OSPINA

321
2
9

I would also recommend async. – Mijago Jan 27 '16 at 14:45

saintedlama · Answer 2 · 2016-01-27T17:48:32.870

Depending on your use case you could combine the two streams into one by using the multistream module.

Multistreams are constructed from an array of streams

var MultiStream = require('multistream')
var fs = require('fs')

var streams = [
  fs.createReadStream(__dirname + '/numbers/1.txt'), // contains a single char '1'
  fs.createReadStream(__dirname + '/numbers/2.txt'), // contains a single char '2'
  fs.createReadStream(__dirname + '/numbers/3.txt')  // contains a single char '3'
]

MultiStream(streams).pipe(process.stdout) // => 123

In case combining streams does not fit the use case you could build your stream on end event sending functionality on your own

const fs = require('fs');

var number1 = fs.createReadStream('./numbers1.txt')
  .on('data', d => console.log(d.toString()));

var number2 = fs.createReadStream('./numbers2.txt')
  .on('data', d => console.log(d.toString()));

onEnd([number1, number2], function(err) {
  console.log('Ended with', err);
});

function onEnd(streams, cb) {
  var count = streams.length;
  var ended = 0;
  var errored = null;

  function shouldEnd() {
    ended++;

    if (errored) { return; }

    if (count == ended) {
      cb();
    }
  }

  function endWithError(err) {
    if (errored) { return; }

    errored = true;
    cb(err);
  }

  streams.forEach(s => s
    .on('end', shouldEnd)
    .on('error', endWithError)
  );
}

The onEnd function can be used to wait for a stream array to end or in case an error event is emitted for the first emitted error event.

score 2 · Answer 3 · answered May 17 '18 at 16:49

Try do it with async functions:

const { createReadStream } = require("fs")

async function main() {
  const stream1 = createReadStream(__dirname + "/1.txt")
  await pipe(stream1, process.stdout)

  const stream2 = createReadStream(__dirname + "/2.txt")
  await pipe(stream2, process.stdout)

  const stream3 = createReadStream(__dirname + "/3.txt")
  await pipe(stream3, process.stdout)
}

async function pipe(tap, sink) {
  return new Promise((resolve, reject) => {
    tap.pipe(sink, { end: false })
    tap.on("end", resolve)
    tap.on("error", reject)
  })
}

zangw · Answer 4 · 2016-01-27T14:48:22.710

Try do it with Promise

function doStream1(cb) {
    // put those operation on stream1 in the callback function...    
    cb && cb();

    var p = new Promise(function(resolve, reject) {
        stream1.on( 'end', resolve );
        stream1.on( 'error', reject );
    });

    return p;
}

function doStream2(cb) {
   // some operation on stream2 on callback 
   cb && cb();

    var p = new Promise(function(resolve, reject) {
        stream2.on( 'end', resolve );
        stream2.on( 'error', reject );
    });

    return p;
}

doStream1(cb).then(function() {
   return doStream2(cb);
}).catch(function(e) {
   // error handling is here
});

score 0 · Answer 5 · answered Jul 05 '19 at 15:35

Try the code below(sequenceStream function). The thing which I am worried about is error handling, but it should work. Also, if you use Node < 10.* then you need end-of-stream instead of stream.finished

const stream  = require('stream');
const process = require('process')
const { promisify } = require('util')

const fromList = (lst) => new stream.Readable({
  read() {
    if (lst.length) {
      this.push(String(lst.shift()))
    } else {
      this.push(null)
    }
  }
})

const _finished = promisify(stream.finished)

const sequenceStream = (streams) => {
  const resultingStream = new stream.PassThrough()
  let isNext = Promise.resolve()
  for(const [i, curStream] of streams.entries()) {
    isNext = isNext.then(() => {
      curStream.pipe(resultingStream, {end: i === streams.length -1})
      return _finished(curStream)
    }).catch((err) => {
      resultingStream.write(err)
    })

  }
  return resultingStream
}

sequenceStream([
    fromList([1, 2, 3, 4]),
    fromList([5, 6, 7, 8]),
    fromList([9, 10])
]).pipe(process.stdout)

How do I run multiple node streams in sequence?

5 Answers5