NodeJS gm hanging when importing images

Question

I have the following script to import raw images from an api:

let importImages = function (meal,accessToken) {

  let imagesData = meal.afbeeldingenInfo;
  let mealFolder = 'client/import/meal-'+meal.maaltijdId;

  ensureExists(mealFolder, 0o755, function(err) {
    if (err) {
      winston.error('Not able to create a folder : %s', err.message);
    } else {
      winston.info('Folder %s succesfully created', mealFolder);
      for (let index in imagesData) {
        if (imagesData.hasOwnProperty(index)) {
          let imageData = imagesData[index];

          axios({
            method: 'get',
            responseType: 'stream',
            url: SYS_URL + '/MaaltijdAfbeelding/' + imageData.id,
            headers: {'Authorization': 'Bearer ' + accessToken},
          }).then(response => {
            winston.info('Got successful response from the API');
            // let image = response.data;
            let filePath = mealFolder + '/'+ imageData.name;
            response.data.pipe(fs.createWriteStream(filePath));
            winston.info('Image %s succesfully imported', imageData.name);
            response.data.on('end', () => {

              // generate images for all sizes
              for (var key in imageSizes) {
                if (imageSizes.hasOwnProperty(key)) {
                  let size = imageSizes[key];
                  gm(filePath).resize(size).strip().interlace('Line')
                    .write(mealFolder + '/' + key + '-' + imageData.name, function (err) {
                      if (!err) {
                        winston.info('Image %s successfully minimized to '+ size +'px', imageData.name)
                      } else {
                        winston.info('Not able to minimize image %s due to error: %s', imageData.name, err.message)
                      }
                    });
                }
              }
            })
          }).catch(error => {
            winston.error('Not able to import image %s due to error: %s', imageData.name, error.message);
          });
        }
      }
    }
  });
};

However when I execute this, after a few seconds the node script freezes the system entirely.

Should this not be done in this manner? If so what is a better practice for resizing images?

If you remove the `.strip()` and `.interlace()` methods, does it run? — Stretch0, Apr 23 '18 at 13:53
@Stretch0 it does the same as before; it runs for about 30 to 40 images and then exponentially gets slower. — xfscrypt, Apr 23 '18 at 14:59
30 - 40 images? So it actually processes some and finishes but then starts to fail after too many? — Stretch0, Apr 23 '18 at 15:14
@Stretch0 yes correct, the system slowly starts to hang as it has processed some of the images (maybe total of 300). — xfscrypt, Apr 23 '18 at 15:41
I would recommend running these processes through a queue. This will mean only one process runs at once but but if any tasks fail, you will be able to reprocess them in the queue. This is a great library for running queues https://www.npmjs.com/package/queue — Stretch0, Apr 23 '18 at 15:56
@Stretch0 ok, i think that might actually help. if you provide an answer, ill mark it as answered. — xfscrypt, Apr 23 '18 at 16:13

Stretch0 · Accepted Answer · 2018-04-24T08:31:17.790

1

Sounds like you are running too many processes at once. I would recommend running these processes through a queue.

This will mean only one process runs at once but but if any tasks fail, you will be able to reprocess them in the queue. This is a great library for running queues

edited Apr 24 '18 at 08:31

answered Apr 23 '18 at 16:19

Stretch0

8,362
13
71
133

NodeJS gm hanging when importing images

1 Answers1