Mongoose JS promises? Or how to manage batch save

Question

How do I manage batch save in Mongoose? I saw it may not be possible yet:

• How can I save multiple documents concurrently in Mongoose/Node.js?

Theres some mention about using some flow control library like q, but I also notice there promises in mongoose, can it be used? Can I do like in jQuery Deferred/Promises

$.when(obj1.save(), obj2.save(), obj3.save()).then ->
    # do something? 

Answer 1

Yes, you can do this with promises. If you were using the Q promise library, you could re-write @matz3's code like:

var tasks = [];

for (var i=0; i < docs.length; i++) {
  tasks.push(docs[i].save());
}

Q.all(tasks)
  .then(function(results) {
    console.log(results);
  }, function (err) {
    console.log(err);
  });

We start all the operations one at a time in the loop, but we don't wait for any of them to complete, so they run in parallel. We add a promise (that acts like a placeholder for the result) to an array. We then wait for all the promises in the array of promises to complete.

Most good Promises/A+ compatible libraries have some equivalent to Q.all

Answer 2

mongoose now allows you to choose which Promise implementation.

Here I am using the node.js default system Promise (ES6) baked into nodejs

var mongoose = require('mongoose');
    mongoose.Promise = global.Promise; // use system implementation

Promise.all(obj1.save(), obj2.save(), obj3.save())
.then(function(resultSaves) {

    console.log('parallel promise save result :');
    console.log(resultSaves);
    mongoose.disconnect();

}).catch(function(err) {

    console.log('ERROR on promise save :');
    console.log(err);
    mongoose.disconnect();
});

node --version v4.1.1

mongoose@4.1.8

Answer 3

Since mongoose now supports promises you may use Promise.all().then(), so it will return when all promises are resolved.

Promise.all([
  obj1.save(),
  obj2.save(),
  obj3.save()
])
.then(console.log)
.catch(console.error)

In fact, if you're always calling the save() method you can use the Array.map() here:

Promise.all([ obj1, obj2, obj3 ].map( obj => obj.save() )

Aaand also use es6 syntax to destructure the resulting array:

Promise.all(
  [ obj1, obj2, obj3 ]
  .map( obj => obj.save() )
)
.then( ([ savedObj1, savedObj2, savedObj3 ]) => {
   // do something with your saved objects...
})

Answer 4

Try the parallel function of the async module.

var functions = [];

for (var i=0; i < docs.length; i++) {
    functions.push((function(doc) {
        return function(callback) {
            doc.save(callback);
        };
    })(docs[i]));
}

async.parallel(functions, function(err, results) {
    console.log(err);
    console.log(results);
});

Answer 5

To save multiple mongoose docs in parallel, you can do something simple like this (assuming you have an array named docs of documents to save):

var count = docs.length;
docs.forEach(function(doc) {
    doc.save(function(err, result) {
        if (--count === 0) {
            // All done; call containing function's callback
            return callback();
        }
    });
});

Answer 6

A refined example on how to use async parallel would be:

  async.parallel([obj1.save, obj2.save, obj3.save], callback);

Since the convention is the same in Mongoose as in async (err, callback) you don't need to wrap them in your own callbacks, just add your save calls in an array and you will get a callback when all is finished.

Answer 7

What about async.queue.
A simple example:

var queue = async.queue(function(obj, callback) {
  return obj.save(callback);
});

for (var i in objs) {
  var obj = objs[i];
  // Some changes on object obj
  queue.push(obj);
}

If you need a callback after the queue is emptied:

var emptyQueue = true;
var queue = async.queue(function(obj, callback) {
  return obj.save(callback);
});
queue.drain = function() {
  // Every callbacks are finished
  // bigCallback();
};

for (var i in objs) {
  var obj = objs[i];
  // Some changes on object obj
  queue.push(obj);
  emptyQueue = false;
}
if (emptyQueue) {
  // Call manually queue drain in case of the queue is empty
  //  and we need to call bigCallback() for example
  return queue.drain();
}

Answer 8

@ForbesLindesay Why loading an external library when you can use mongoose implementation of promises and create your own All ?

Create a module that enhance mongoose promise with all.

var Promise = require("mongoose").Promise;

Promise.all = function(promises) {
  var mainPromise = new Promise();
  if (promises.lenght == 0) {
    mainPromise.resolve(null, promises);
  }

  var pending = 0;
  promises.forEach(function(p, i) {
    pending++;
    p.then(function(val) {
      promises[i] = val;
      if (--pending === 0) {
        mainPromise.resolve(null, promises);
      }
    }, function(err) {
      mainPromise.reject(err);
    });
  });

  return mainPromise;
}

module.exports = Promise;

Then use it with mongoose:

require('./promise')

...

var tasks = [];

for (var i=0; i < docs.length; i++) {
  tasks.push(docs[i].save());
}

mongoose.Promise.all(tasks)
  .then(function(results) {
    console.log(results);
  }, function (err) {
    console.log(err);
  });