0

My ampq system seems loosing messages, so I'd like a way to see if messages are effectively queued before being consumed.

I have several MicroServices communicating by amqp messages on NodeJs, using CloudAmqp. One of this microservice MS[B] generates .pdf, the process it's pretty heavy and requires about a minute for each request. So I send the .pdf asyncronously, triggering a webhook once finished, and generate once per time using a PreFetch = 1

So one MS[A] collects all the requests from the user, answers back to them saying "ok, request received, listen on the webhook" and in parallel it asks to the MS[B] to generate pdfs. MS[B] has prefetch=1, so consumes just one request per time. Once finished, sends the response to the callback queue of MS[A], which triggers the user webhook saying "the pdf, it's ready".

The problem is that MS[B] misses all the messages while busy:

  1. it consumes one request from MS[A]
  2. starts generating the .pdf
  3. while generating, it discards all the other messages that MS[A] sends, as if there would be not any queue
  4. it finishes the .pdf, sending ACK to MS[A]
  5. then it starts again accepting messages, taking the last one received after being idle, losing all the previous ones.

Why? How can I find the problem, what could I monitor?

Communications between other MSs works well, with messages correctly ordered in queues. Just this one, with prefetch=1, loses messages.

I am NOT using the NO-ACK rule. I don't know what try, what test and what monitor to find the problem.

How can I see (if) messages are correctly queued before being consumed, ora just lost?

Below, the implementation of the messaging system

Channel Creation

      /* 
         Starting Point of a connection to the CLOUDAMQP_URL server, then exec the callback
        */
      start(callback) {
        var self = this;
        // if the connection is closed or fails to be established at all, we will reconnect
        amqp.connect(process.env.CLOUDAMQP_URL + "?heartbeat=60")
          .then(
            function (conn) {
              // create queues and consume mechanism
              self.amqpConn = conn;
              setTimeout(() => {
                startPublisher();
              }, 200);
              setTimeout(() => {
                createCallbackQueue();
              }, 1000);
              setTimeout(() => {
                callback();
              }, 2000);
            });

        // create publisher channels
        function startPublisher() {
          self.amqpConn.createConfirmChannel()
            .then(function (ch) {
              self.pubChannel = ch;
              logger.debug("️ pubChannel ready");
              while (true) {
                var m = self.offlinePubQueue.shift();
                if (!m) break;
                self.publish(m[0], //  exchange
                  m[1], //  routingKey
                  m[2], //  content,
                  undefined //  correlationId
                );
              }
            });
        }

        // create callback channel
        function createCallbackQueue() {
          self.amqpConn.createChannel()
            .then(function (channel) {
              channel.assertQueue(self.CALLBACK_QUEUE_NAME, {
                  durable: true,
                  exclusive: true, // callback are exclusive
                })
                .then(function (q) {
                  logger.debug("  Waiting for RPC RESP in " + self.CALLBACK_QUEUE_NAME);
                  channel.consume(q.queue,
                    processCallback, {
                      noAck: false
                    }
                  );
                });

              // process messages of the callback
              function processCallback(msg) {
                var correlationId = msg.properties.correlationId;
                }
                //callback received 
                if (self.correlationIds_map[correlationId]) {
                  delete self.correlationIds_map[correlationId];
                  var content = JSON.parse(msg.content.toString());
                  self.eventEmitter.emit(correlationId, content);

                }

              }
            });
        }
        return deferred.promise;
      }  

Consuming Messages

   /*
    @worker_queue - the name of the queue
    */
   // Consume message from 'worker_queue', A worker that acks messages only if processed succesfully
   startWorker(worker_queue, routes) {
     var self = this;
     logger.debug("startWorker " + self.CALLBACK_QUEUE_NAME);
     var channel;
     worker_queue = self.MICROSERVICE_NAME + worker_queue;
     self.amqpConn.createChannel()
       .then(
         function (ch) {
           channel = ch;
           ch.prefetch(self.opt.prefetch); // = 1 for MS[B] generating pdf
           channel.assertQueue(worker_queue, {
               durable: true,
               exclusive: true
             })
             .then(function (q) {
               channel.consume(worker_queue, processMsg, {
                 noAck: false
               });
             });
         });



     //  call the 'function from interface' passing params, and send the ACK
     function processMsg(msg) {
       work(msg)
         .then(function (data) {
           channel.ack(msg, false); // allUpTo = false
         })
         .catch(function (err) {
           channel.ack(msg, false);
           // channel.reject(msg, false); // requeue = false
           // this.closeOnErr(e);
         });
     }

     //  execute the command, and queue back a response, checking if it's an error or not
     function work(msg) {
       var deferred = Q.defer();
       var correlationId;
       try {
         correlationId = msg.properties.correlationId;
       } catch (err) {}
       work_function(msg.content, correlationId)
         .then(function (resp) {
           var content = {
             data: resp
           };
           content = Buffer.from(JSON.stringify(content));
           channel.sendToQueue(msg.properties.replyTo,
             content, {
               correlationId: correlationId,
               content_type: 'application/json'
             }
           );
           deferred.resolve(resp);
         });
       return deferred.promise;
     }
   }

Publish Messages

publish(exchange, routingKey, content, correlationId) {
  var self = this;
  var deferred = Q.defer();


  self.correlationIds_map[correlationId] = true;
  self.pubChannel.publish(exchange, routingKey, content, 
    {
      replyTo: self.CALLBACK_QUEUE_NAME,
      content_type : 'application/json',
      correlationId: correlationId,
      persistent   : true
    },
    function(err, ok) {
      if (err) 
          {
              self.offlinePubQueue.push([exchange, routingKey, content]); // try again
              self.pubChannel.connection.close();
              deferred.resolve('requeued');
          }
      else
      { 
       deferred.resolve(ok);
      }
  });
return deferred.promise;
}
DeLac
  • 1,068
  • 13
  • 43
  • I think it's something related to the `expiration`field. At a certain point, debugging messaging, it's `0`. I should understand if it's due to time lapsing, or to a wrong configuration – DeLac Jan 20 '23 at 16:50

0 Answers0