async.series nested in async.eachSeries loop terminates early sending POST response

Question

I am on the recieving end of a POST call which contains an array of objects (requestArray). Before I respond to the POST, I need to pass the array's objects through a series of functions, in sequence. I chose the async library to help me with this task however I'm having difficulty controlling the code's execution flow.

I am using a global array to store the results of each function (responseArray). Some of the functions depend on the result of prior functions. I don't want to use async.waterfall() because 1. I'll have to rewrite my code and 2. I may encounter the same early loop termination issue. Below is my code problematic code.

app.post('/test', function(req, res) {
  var importArray = req.body;
  var iteration = 0;

  async.eachSeries(importArray, function(data, callback) {
    var index = importArray.indexOf(data);
    var element  = data.element;
    exportArray[index] = [];

    async.series([
      function(callback) {
        process1(data, index, callback);
      },
      function(callback) {
        process2(element, index, callback);
      },
      function(callback) {
        process3(element, index, callback);
      }],
      function(err, results) {  
        var str = {};
        results.forEach(function(result) {
          if (result) {
              str += result + ',';
          }
        });
        //callback();       // callback here = synchronous execution.
        if (index === req.body.length - 1) {
          res.send(exportArray);
        }
      });
      console.log('async.eachSeries() callback iteration # = ' + iteration);
      iteration++;
      callback();           // callback here = asynchronous execution.
  }, function(err){
      if( err ) {
        console.log('Error');
      } else {
        console.log('All data has been processes successfully.');
      }
  });
});

Each function in the async.series() returns callback(null, result). After process1() returns its callback, the async.eachSeries() jumps to the next array entry before, which is ideal. However, async.eachSeries() executes a POST response before all of the async.series() results are returned. How could I revise my code so that async.eachSeries() finishes execution after all of the importArray results (exportArray) are returned from process1 - 3 before I send a POST response?

Being a little older and wiser I realize the problem with early `async.eachSeries()` loop termination is due my mistake of inserting the asynchronous method `results.forEach()` at the end of the `async.series()` method. Instead of `results.forEach()`, I should've used a synchronous method like a `for()` loop. — coolcat44, Aug 18 '19 at 03:30

Bulkan · Answer 1 · 2015-05-15T00:59:16.063

I suggest for ease of following the async nature of your code to rename your callbacks slightly. Also to wait until eachSeries has finished move res.send to eachSeries final callback passing in the results.

Here is the updated code.

app.post('/test', function(req, res) {
  var importArray = req.body;
  var iteration = 0;

  async.eachSeries(importArray, function(data, next) {
    var index = importArray.indexOf(data);
    var element  = data.element;
    exportArray[index] = [];

    async.series([
      function(cb) {
        process1(data, index, cb);
      },
      function(cb) {
        process2(element, index, cb);
      },
      function(cb) {
        process3(element, index, cb);
      }],

      function(err, results) {  
        var str = {};
        results.forEach(function(result) {
          if (result) {
              str += result + ',';
          }
        });

        console.log('async.eachSeries() callback iteration # = ' + iteration);
        iteration++;
        next(null, results);      
      });     

  }, function(err, results){
      if(err) {
        return console.log('Error');
      } 

      res.send(exportArray);
      console.log('All data has been processes successfully.');
  });
});

Thanks for revising, Bulkan. `next(null, results)` is throwing an error because `results` is not in scope. Also, should I have 2 `res.send(exportArray)` statements? — coolcat44, May 14 '15 at 12:59
I was able to implement your changes and it works correctly. However, it does run synchronously. I'm copying my code that I got to run asynchronously. — coolcat44, May 18 '15 at 21:38

score 1 · Answer 2 · answered May 18 '15 at 21:52

After help from @Bulkan and tinkering, I got the code to run asynchronously with the help of my old friend, "flag". Here's the code:

app.post('/test', function(req, res) {
  var importArr = req.body;
  var iteration = 0;
  var flag      = false;

  async.eachSeries(importArr, function(data, cb) {
    var index        = importArr.indexOf(data);
    var element      = data.element;
    exportArr[index] = [];

    async.series([
      function(cb) {
        process1(data, index, cb);
      },
      function(cb) {
        process2(element, index, cb);
      },
      function(cb) {
        process3(element, index, cb);
      }],

      function(err, results) {  
        var str = {};
        results.forEach(function(result) {
          if (result) {
              str += result + ',';
          }
        });
        iteration++;
        if (iteration === req.body.length) {
          flag = true;
          res.send(exportArr);
        } 
      });
      console.log('async.eachSeries() callback iteration # = ' + iteration);

      if (iteration < req.body.length) {
        cb();
      } else if (flag)  {
        cb();
      }
  }, function(err){
      if (err) {
        console.log('Error');
      } else {
        console.log('All data has been processes successfully.');
      }
  });
});

async.series nested in async.eachSeries loop terminates early sending POST response

2 Answers2