How to handle async loop?

Question

I'm currently working on a project to implement a full-text search client-side in JavaScript based on lunr.js.

The thing is, I'm struggling in building then saving the index because I have several asynchronous calls.

    function buildIndex(rawIndex, root, indexPath = root + 'js/app/index.json') {
  var path = path || require('path');
  var fs = fs || require('fs'),
    promesses = [],
    ignore = ['node_modules'],
    files = fs.readdirSync(root);
  files.forEach(function (file) {

    if (fs.statSync(path.join(root, file)).isDirectory() && ignore.indexOf(file) == -1) {
      buildIndex(rawIndex, path.join(root, file), indexPath);
    }
    else if (file.substr(-5) === '.html' && file != 'example.html') {
      var promesse = JSDOM.fromFile(path.join(root, file)).then(dom => {

        var $ = require('../lib/_jquery')(dom.window);
        populate();
        console.log(file + " indexé");
        function populate() {
          $('h1, h2, h3, h4, h5, h6').each(function () {
            var title = $(this);
            var link = path.join(root, file).replace('..\\', '') + "#" + title.prop('id');
            var body = title.nextUntil('h1, h2, h3, h4, h5, h6');
            rawIndex.add({
              id: link,
              title: title.text().latinise(),
              body: body.text().latinise()
            });
          });
        };
      });
      promesses.push(promesse);
    }
  });
  Promise.all(promesses)
    .then(function () {
      fs.writeFileSync(indexPath, "var data = " + JSON.stringify(rawIndex), 'utf8');
    })
    .catch(function (err) {
      console.log("Failed:", err);
    });
};

Thanks in advance.

You're not awaiting the result of the recursive call - it doesn't return a promise, and you don't put it in your array. — Bergi, Jul 27 '17 at 16:36
@JamshidAsadzadeh Well, I don't think so, at least it is not mentionned in the docs : https://lunrjs.com/docs/lunr.Builder.html. I would rather say the problem is linked to _$('h1, h2, h3, h4, h5, h6').each()_ and loop within loop. — DnzzL, Jul 28 '17 at 08:44
@DnzzL Your edit to [revision](https://stackoverflow.com/posts/45354163/revisions) 3 made the code significantly worse. I wouldn't mind if you rolled it back. — Bergi, Jul 28 '17 at 14:13

score 1 · Answer 1 · answered Jul 28 '17 at 14:12

There are four problems:

Your function buildIndex does not return the promise, so one cannot wait for the result when calling it
When encountering a directory, you call buildIndex recursively but don't try to wait for its result like you did with promesse in the else case.
You have the promesses.push(promesse); call inside the asynchronous callback that executes only after the file was read in. The idea of putting the promise in the array is right, but you must do it immediately so that it happens before Promise.all is called upon the array.
You for some reason removed Promise.all from the code.

Basically the function should have this general schema:

function buildIndex(…) {
  …
  var promises = paths.map(function(path) {
    if (isDir(path)) {
      return buildIndex(…);
    } else {
      return JSDOM.fromFile(…).then(…);
    }
  });
  return Promise.all(promises).then(…);
}

Thank you very much for your time from a new JS learner, it seems to work very well. — DnzzL, Jul 31 '17 at 06:44

score 0 · Accepted Answer · answered Jul 31 '17 at 06:50

Using forEach wasn't the right choice to make as one wants to return a Promise. Thus it is wiser to make use of .map and then return Promises in the if/else statement. Finally, one has to call Promises.all(promises) making .then(...) usable as expected.

My final function:

function buildIndex(rawIndex, root, indexPath = root + 'js/app/index.json') {
  var path = path || require('path');
  var fs = fs || require('fs'),
    promises = [],
    ignore = ['node_modules'],
    files = fs.readdirSync(root);

  var promises = files.map(function (file) {
    if (fs.statSync(path.join(root, file)).isDirectory() && ignore.indexOf(file) == -1) {
      return buildIndex(rawIndex, path.join(root, file), indexPath);
    }
    else if (file.substr(-5) === '.html' && file != 'example.html') {
      return JSDOM.fromFile(path.join(root, file)).then(dom => {

        var $ = require('jquery')(dom.window);
        populate();
        console.log(file + " indexé");

        function populate() {
          $('h1, h2, h3, h4, h5, h6').each(function () {
            var title = $(this);
            var link = path.join(root, file).replace('..\\', '') + "#" + title.prop('id');
            var body = title.nextUntil('h1, h2, h3, h4, h5, h6');
            rawIndex.add({
              id: link,
              title: title.text().latinise(),
              body: body.text().latinise()
            });
          });
        };
      })
    }
  })
  return Promise.all(promises).then(function () {
    fs.writeFileSync(indexPath, "var data = " + JSON.stringify(rawIndex), 'utf8');
  });
};

Thanks @Bergi for the answer and those who helped.

How to handle async loop?

2 Answers2