How to exit nodejs at the end of the stream, so last values are saved to db, and log properly

Question

I have a baconjs stream I use to save articles with mongoose. Here's the code:

function main() {
  db.once('open', function(callback) {
    console.log('db connection successs');
    console.log('stremaing page ' + page);

    var stream = readSite(page);
    stream.onValue(function(article) {
      try {
        article = toArticleModel(article);
      } catch (e) {
        console.error(e);
      }
      article.save(function(err, article) {
        if (err) {
          console.error('Save Error: ' + err);
        } else {
          console.log('saved ' + article.publishedDate + " " + article.author.name + " " + article.title);
        }
      });

      stream.onError(function(err) {
        console.error('Stream Error: ' + err);
      });

      stream.onEnd(function() {
        console.log('stream ended closing in 15 seconds..');
        setTimeout(function() {
          db.close();
        }, 15 * 1000);
      });
   });
 });

Once the stream ends onEnd, I want to close the db connection and exit the nodejs program. I figured once the stream ends I should wait for some time so the latest values gets saved to the db. So I use setTimeout for 15 seconds and use db.close.

The problem is, the log stream ended closing in 15 seconds.. is logged 50-60 times in the stdout, Why? And is this a good approach to exit such a program?

score 1 · Accepted Answer · answered Jan 07 '15 at 21:50

You are adding the stream.onEnd listener inside of the stream.onValue. This means that you are adding the listener each time the onValue is triggered. You need to add the event listeners outside of other event listeners so they are only added once.

There are a variety of methods to handle asynchronous iteration to make sure all article values get saved before closing the DB (a step you may actually be able to get away with just skipping).

// initialize savingArticles to 0

savingArticles++;
article.save(function (err, article) {
  savingArticles--
  if (streamEnded && !savingArticles)
    db.close();

stream.onEnd(function () {
  if (!savingArticles)
    db.close();
  else
    streamEnded = true;

This will close the DB if an article is not currently being saved and the stream has ended. If the stream ends while an article is currently being saved, then the DB should be closed after that article is saved.

Using mutable state (variables `savingArticles`, `streamEnded`) is a sign of bad code when using FRP. Using `flatMap` and `fromNodeCallback` in Roman Pominov's answer is far better. — OlliM, Jan 13 '15 at 13:17

score 1 · Answer 2 · edited Jan 13 '15 at 14:09

I'd try to use .fromNodeCallback and .flatMap for this:

stream = stream.flatMap(function(article) {

  try {
    article = toArticleModel(article);
  } catch (e) {
    return new Bacon.Error('toArticleModel error: ' + err);
  }

  return Bacon.fromNodeCallback(function(callback) {
    article.save(function(err, article) {
      if (err) {
        callback('Save Error: ' + err);
      } else {
        callback('saved ' + article.publishedDate + " " + article.author.name + " " + article.title);
      }
    });
  });

});

stream.onError(console.error.bind(console));
stream.onValue(console.log.bind(console));
stream.onEnd(db.close.bind(db));

It also can be simplified further:

stream = stream.flatMap(function(article) {

  try {
    article = toArticleModel(article);
  } catch (e) {
    return new Bacon.Error(err);
  }

  return Bacon.fromNodeCallback(article, "save")
    .map(function() {
      return "saved " + article.publishedDate + " " + 
        article.author.name + " " + article.title;
    });

});

stream.log();
stream.onEnd(db.close.bind(db));

How to exit nodejs at the end of the stream, so last values are saved to db, and log properly

2 Answers2