Uncatchable MQTT aws-crt-nodejs crash

Question

I am using the MQTT client in aws-crt to subscribe and publish, but every once in a while the server crashes with an error I have not been able to catch in JavaScript, but seems to be thrown in the C libraries of the package.

Does anyone have any idea of how I can catch/prevent this? Am I doing something I am not supposed to be doing? Has anyone else come across this issue?

I have not been able to replicate this on purpose, but it seemingly happens some hours after the server starts up.

The server is an EC2 instance running Amazon Linux 2.

I have made a wrapper class, and every call to its methods is within a try/catch block everywhere they are used:

aws-shadow-client.js:


class ShadowClient {

    /* ... */

    subscribe(serialNumber, callback, qos = this.#QoS) {
        try {
            return this.#subClient.subscribe(
                `$aws/things/${serialNumber}/shadow/update/accepted`,
                qos,
                (topic, message, dup, qos, retain)=> {
                    message = JSON.parse(textDecoder.decode(message));
                    callback(message, topic);
                }
            );
        } catch(error) {
            console.log('Could not set up AwsShadowClient subscribe.', error);
            throw error;
        }
    }

    publish(serialNumber, payload, qos = this.#QoS) {
        try {
            return this.#pubClient.publish(
                `$aws/things/${serialNumber}/shadow/update`,
                payload,
                qos
            );
        } catch(error) {
            console.log('Could not publish AwsShadowClient', error);
            throw error;
        }
    }

    unsubscribe(serialNumber) {
        try {
            return this.#subClient.unsubscribe(`$aws/things/${serialNumber}/shadow/update`);
        }
        catch (error) {
            console.log('Could not unsubscribe AwsShadowClient', error);
            throw error;
        }
    }
}

These are called in websocket-server.js:

webSocketServer.on('connection', async (centralConnectionSocket) => {
    const UUID = uuid();
    let logTag = `[SN: ${UUID}]`;
    let deviceName = '';

/* ... Set deviceName among other things */

    centralConnectionSocket.on('close', (code, reason = '[empty]') => {
        console.log(logTag, `Disconnected from central server. Stopping active script.\nDisconnect info:\nCode: ${code}\nReason: '${reason}'`);

        stopEverything()
        .catch(() => {
            console.log(logTag, 'Failed to stop everything');
        })
        .finally(async ()=>{
            try {
                try {
                    // WARNING: We need to make sure the instrument closes the tunnel. Otherwise, everything breaks
                    const shadowClient = await AwsShadowClient.factory();

                    // Unsub to clear the previous disconnect and error handlers
                    await shadowClient.unsubscribe(deviceName);

                    await shadowClient.publish(deviceName, {
                        state: {
                            desired: {
                                uuid: UUID, myMsg: 'some msg'
                            }
                        }
                    });
                }
                catch (error) {
                        console.log(logTag, 'Failed in MQTT workflow. Forcing port release anyway.\n', error.message);
                    }
                    finally {
                        releasePort();
                    }
                }
                finally {
                    // ... Do some more clean up
                    }
                }
            });
        });
}

And this is what is written to the log file:

[SN: 0000-0000] Failed to stop everything
Signal received: 1742688112, errno: 32561
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
Signal received: 0, errno: 0
################################################################################
Stack trace:
################################################################################
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(+0x6e903) [0x7f31741df903]
/lib64/libpthread.so.0(+0x118e0) [0x7f31781578e0]
/lib64/libgcc_s.so.1(+0xe407) [0x7f3178372407]
/lib64/libgcc_s.so.1(_Unwind_Backtrace+0x62) [0x7f3178373e52]
/lib64/libc.so.6(backtrace+0x78) [0x7f3177e95728]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace+0x1b) [0x7f31743ec6db]
/var/app/current/node_modules/aws-crt/dist/bin/linux-x64/aws-crt-nodejs.node(aws_backtrace_print+0x41) [0x7f31743ec931]

I suggest you raised an issue on github. AWS SDK team should be able to support you. — brushtakopo, Jan 23 '23 at 08:51
Thanks for the suggestion. I've opened an issue with them, but it seems I'll have to do some sleuthing myself (it's not like the stack trace is very helpful in this instance) — Samuel Poveda, Jan 23 '23 at 22:10

Uncatchable MQTT aws-crt-nodejs crash

0 Answers0