9

So I have a fairly simple setup on Heroku. I'm using RabbitMQ for handling background jobs. My setup consists of a node script that runs daily using Heroku Scheduler addon. The scripts adds jobs to the queue, the worker in turn, consumes them and delegates them onto a separate module for handling.

The problem starts after I receive a SIGTERM event that Heroku initiates randomly from time to time, before restarting the instance.

For some reason, after the instance is restarted, the worker is never get back up again. Only when I restart it manually by doing heroku ps:scale worker=0 and heroku ps:scale worker=1 The worker continues to consume the pending jobs.

Here's my worker:

// worker.js
var throng = require('throng');
var jackrabbit = require('jackrabbit');
var logger = require('logfmt');
var syncService = require('./syncService');

var start = function () {
    var queue = jackrabbit(process.env.RABBITMQ_BIGWIG_RX_URL || 'amqp://localhost');

    logger.log({type: 'msg', msg: 'start', service: 'worker'});

     queue
        .default()
        .on('drain', onDrain)
        .queue({name: 'syncUsers'})
        .consume(onMessage)

    function onMessage(data, ack, nack) {

        var promise;
        switch (data.type) {
            case 'updateUser':
                promise = syncService.updateUser(data.target, data.source);
                break;
            case 'createUser':
                promise = syncService.createUser(data.source);
                break;
            case 'deleteUser':
                promise = syncService.deleteUser(data.target);
        }

        promise.then(ack, nack);
    }

    function onDrain() {
        queue.close();
        logger.log({type: 'info', msg: 'sync complete', service:    'worker'});
     }

    process.on('SIGTERM', shutdown);


    function shutdown() {
        logger.log({type: 'info', msg: 'shutting down'});
        queue.close();
        process.exit();
    }

};


throng({
    workers: 1,
    lifetime: Infinity,
    grace: 4000
}, start);
yohairosen
  • 1,665
  • 1
  • 18
  • 26
  • what are you seeing in the log files from heroku, after the SIGTERM happens? are you getting the "shutting down" message? what about on the RMQ side - after SIGTERM shuts down the heroku process, does RMQ show messages in "unacked" state? are you seeing connections and channels still open? – Derick Bailey Sep 22 '16 at 03:29
  • No "shutting down" message just "Stopping all processes with SIGTERM" then "Process exited with status 0". – yohairosen Sep 26 '16 at 10:57

1 Answers1

1

The close() method on the jackrabbit object takes a callback, you should avoid exiting the process until that is finished:

function shutdown() {
    logger.log({type: 'info', msg: 'shutting down'});
    queue.close(function (e) {
      process.exit(e ? 1 : 0);
    });
}
idbehold
  • 16,833
  • 5
  • 47
  • 74