RabbitMQ reconnect logic is not working on abrupt restart of RabbitMQ server

Question

My node application and RabbitMQ server are running on docker.
That nodejs application have reconnect logic which works great when I gracefully restart RabbitMQ service by kill command in RabbitMQ container or by sudo docker-compose restart -t 10 rabbitmq which wait till 10 seconds before sending forcefull kill singal to RabbitMQ service.

But when I forcefully restart RabbitMQ service by sudo docker-compose restart -t 0 rabbitmq then reconnect logic does not work. lsof command on nodejs app container does not show any connections.

var amqp = require('amqplib')
var appRoot = require('app-root-path')
var rabbitMQConfig = require(`${appRoot}/config`).rabbitmq
var winston = require(`${appRoot}/logger`)
var utils = require(`${appRoot}/utils`)
var util = require('util')
var rabbitMQController = require(`${appRoot}/controllers/rabbitMQController`)
var request = require('request-promise')

function getRabbitMQConnectionString(rabbitMQConfig) {
    var connPrefix = rabbitMQConfig.prefix
    if (rabbitMQConfig.authEnabled === true) {
        var credentials = rabbitMQConfig.userName + ":" + rabbitMQConfig.password +"@"
        connPrefix += credentials
        connString = util.format(connPrefix +'%s:%s%s', rabbitMQConfig.host, rabbitMQConfig.port, rabbitMQConfig.vhost)
    } else {
        connString = util.format(connPrefix +'%s:%s%s', rabbitMQConfig.host, rabbitMQConfig.port, rabbitMQConfig.vhost)
    }
    return connString   
}

var connString = getRabbitMQConnectionString(rabbitMQConfig)

var initializeRabbitMQ = async function(channel) {
    await channel.assertExchange(rabbitMQConfig.exchange, 'direct', { durable: true })
    await channel.assertQueue(rabbitMQConfig.queue, { durable: true })
    var url = util.format('%s%s:%s/api/bindings/%s/e/%s/q/%s', rabbitMQConfig.httpPrefix, rabbitMQConfig.host, rabbitMQConfig.managementPort, encodeURIComponent(rabbitMQConfig.vhost), rabbitMQConfig.exchange, rabbitMQConfig.queue) 
    var options = {
        "method": "GET",
        "uri": url,
        "headers": {
            "Authorization": "Basic " + new Buffer(rabbitMQConfig.userName + ":" + rabbitMQConfig.password).toString("base64"),
        },
        'json': true,
        'timeout': 15000
    }
    var result = await request(options)
    // console.log('alok', result)
    result.forEach(async function(item) {
        await channel.unbindQueue(rabbitMQConfig.queue, rabbitMQConfig.exchange, item.routing_key)
    })
    rabbitMQConfig.source.forEach(async function(item) {
        await channel.bindQueue(rabbitMQConfig.queue, rabbitMQConfig.exchange, item)
    })
}

var connect = async function() {
    var conn = await amqp.connect(connString)
    var channel =  await conn.createChannel()
    await initializeRabbitMQ(channel)
    rabbitMQController.processMessages()
    return channel
}

var prepareExports = async function(exportsObj) {
    try {
        exportsObj["connection"] = connect()
        var channel = await exportsObj["connection"]
        channel.on('error', function(error) {
            winston.error(utils.getLogString("rabbitmq_channel_error", "", "", error.stack))
        })
        channel.on('close', function() {
            winston.error(utils.getLogString("rabbitmq_channel_close", "", "", ""))
            prepareExports(exportsObj)
        })
    } catch (error) {
        winston.error(utils.getLogString(arguments.callee.name, "", "", error.stack))
        await new Promise((resolve) => {
            setTimeout(resolve, 5000)
        })
        prepareExports(exportsObj)
    }
}

prepareExports(module.exports)

What should be proper way to make it work even in case of forceful abrupt restart of RabbitMQ service?

I could be missing something, but I don't see any reconnect logic in this code. — shkaper, Aug 28 '19 at 18:59
@shkaper: see last line of code `prepareExports(module.exports)` — Alok, Aug 29 '19 at 03:10

score 2 · Answer 1 · answered Sep 02 '19 at 04:03

A couple of tests with a local instance of RabbitMQ showed that a connection.on('close') listener is always being fired when RabbitMQ restarts, so I suggest you try a Connection listener instead of Channel.

While it's true that the library will emit close events for all Channels on a closed Connection, there is no reason to only listen to Channel events (as opposed to Connection events). Moreover, a couple of benefits of listening to Connection events:

If a Channel emits error and there is no corresponding listener, the error will propagate to an underlying Connection.
If a Connection emits close, the listener is called with a reason in the error message. The Connection emits close events for all its Channels without specifying a reason.

score 1 · Answer 2 · answered Sep 04 '19 at 07:03

On error in connection there should be error in chennel also which is not getting raise, its a bug in amqplib. So fix is to register a function on error in connection.

var connect = async function() {
    var conn = await amqp.connect(connString)
    var channel =  await conn.createChannel()
    await initializeRabbitMQ(channel)
    conn.on('error', () => {}) // this is solving problem
    return channel
}

var prepareExports = async function(exportsObj) {
    try {
        exportsObj["connection"] = connect()
        var channel = await exportsObj["connection"]
        channel.on('error', function(error) {
            winston.error(utils.getLogString("rabbitmq_channel_error", "", "", error.stack))
        })
        channel.on('close', function() {
            winston.error(utils.getLogString("rabbitmq_channel_close", "", "", ""))
            prepareExports(exportsObj)
        })
    } catch (error) {
        winston.error(utils.getLogString(arguments.callee.name, "", "", error.stack))
        await new Promise((resolve) => {
            setTimeout(resolve, 5000)
        })
        prepareExports(exportsObj)
    }
}

prepareExports(module.exports)
rabbitMQController.processMessages()

score 0 · Answer 3 · answered Sep 02 '19 at 07:45

connection.close() is available in official api at http://www.squaremobius.net/amqp.node/channel_api.html#model_close

In addition to listening to channel we have to listen to connection also.

var connect = async function() {
    var conn = await amqp.connect(connString)
    var channel =  await conn.createChannel()
    channel.conn_ = conn
    await initializeRabbitMQ(channel)
    // rabbitMQController.processMessages()
    return channel
}

var prepareExports = async function(exportsObj) {
    try {
        exportsObj["connection"] = connect()
        var channel = await exportsObj["connection"]
        var conn = channel.conn_
        channel.on('error', function(error) {
            winston.error(utils.getLogString("rabbitmq_channel_error", "", "", error.stack))
        })
        channel.on('close', function() {
            winston.error(utils.getLogString("rabbitmq_channel_close", "", "", ""))
            prepareExports(exportsObj)
        })
        conn.on('error', function(error) {
            winston.error(utils.getLogString("rabbitmq_conn_error", "", "", error.stack))
        })
        conn.on('close', function() {
            winston.error(utils.getLogString("rabbitmq_conn_close", "", "", ""))
            prepareExports(exportsObj)
        })
    } catch (error) {
        winston.error(utils.getLogString(arguments.callee.name, "", "", error.stack))
        await new Promise((resolve) => {
            setTimeout(resolve, 5000)
        })
        prepareExports(exportsObj)
    }
}

prepareExports(module.exports)
rabbitMQController.processMessages()

RabbitMQ reconnect logic is not working on abrupt restart of RabbitMQ server

3 Answers3