NodeJS heap out of memory

Question

I get an error when sending 5K of emails using the MailjetApi.

My code :

  function arraySend(array, locals) {
    var mailjetConnect = mailjet.connect(locals.blocks.mailjetparam.username, locals.blocks.mailjetparam.password),
        chunkedArrayMessages = _.chunk(array, 50),
        delay = ms => new Promise(resolve => setTimeout(resolve, ms)),
        promises;

    if (process.env.NODE_ENV.trim() === 'dev') {
      return Promise.resolve();
    } else {
      promises = chunkedArrayMessages.map((messages) => {
        return Promise.all([
          delay(1000),
          mailjetConnect.post('send', { version: 'v3.1' }).request({ Messages: messages }).then(() => {
            return Promise.resolve();
          }).catch(() => {
            return Promise.resolve();
          })
        ]);
      });
      return Promise.all(promises).then(function() {
        return Promise.resolve();
      }).catch(function(err) {
        return Promise.reject(err);
      });
    }
  }

Errors :

  <--- Last few GCs --->
  [30337:0x51226d0]    59607 ms: Mark-sweep (reduce) 4085.3 (4103.8) -> 4085.1 (4107.6) MB, 71.7 / 0.1 ms  (+ 79.5 ms in 40 steps since start of marking, biggest step 78.8 ms, walltime since start of marking 255 ms) (average mu = 0.852, current mu = 0.779) [30337:0x51226d0]    59810 ms: Mark-sweep (reduce) 4087.0 (4108.0) -> 4086.9 (4109.0) MB, 119.3 / 0.1 ms  (average mu = 0.781, current mu = 0.411) allocation failure scavenge might not succeed
  <--- JS stacktrace --->
  FATAL ERROR: MarkCompactCollector: young object promotion failed Allocation failed - JavaScript heap out of memor

My NodeJS application is started using PM2 and process.json.

Command that I use to start my NodeJS application : sudo pm2 start process.json --node-args="--max-old-space-size=4096"

My process.json of PM2 :

{apps:[{name:"*****.fr:3006",cwd: "****/mywebsite/cms",script: "./index.js",watch:false, env: {"NODE_ENV": "prod"}}]}

Package.json :

"start": "set NODE_ENV=prod & node index.js",

My server :

Can't you just increase the --max-old-space-size=4096? That seems to be what the error message says. — tom, Jul 31 '23 at 07:18
Promises and array.map use huge amounts of memory. I would suggert either increasing the heap size, reducing the batch size or at least avoiding duplicaitng memory with map and using a simple for loop. — Joel Imbergamo, Jul 31 '23 at 07:20
Could you provide me an edit of my code with your idea ? I need to add a delay of 1000 ms each 50 e-mails (MailjetApi prerequisites). — tonymx227, Jul 31 '23 at 07:44

score 0 · Answer 1 · answered Jul 31 '23 at 06:55

0

You could reduce the batch size from 5K to 1K and queue the batches in Apache Kafka / RabbitMQ.

answered Jul 31 '23 at 06:55

Denis Nutiu

1,178
15
22

score 0 · Answer 2 · answered Jul 31 '23 at 08:29

Due to the limitations of Mailjet, you will not be able to get around an asynchronous function. Try this approach. You may have to adjust some parts.

// maybe better in global context (?)
const mailjetConnect = mailjet.connect(locals.blocks.mailjetparam.username, locals.blocks.mailjetparam.password);
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));

async function arraySend(array, locals) {
  let chunkedArrayMessages = _.chunk(array, 50);
  let result;

  if (process.env.NODE_ENV.trim() === 'dev') {
    return true; // or false (?)
  }
  
  for (let chunkedArray of chunkedArrayMessages) {
    let messages = [];
    for (let chunk of chunkedArray) {
      messages.push(chunk.messages); // structure of chunkedArrayMessages ?
    }
    result = await mailjet
      .post('send', { version: 'v3.1' })
      .request(messages);
    // result validation !!! 
    await delay(1000);
  }

  return true; // or false (?)
}

NodeJS heap out of memory

2 Answers2