nodejs application kafka communication over the http_proxy https_proxy proxy getting error

Question

{"level":"ERROR","timestamp":"2023-08-31T15:17:48.211Z","logger":"kafkajs","message":"[Connection] Connection timeout","broker":"confluent-cloud-kafka:9092","clientId":"lloydsbank-group"} {"level":"ERROR","timestamp":"2023-08-31T15:17:48.212Z","logger":"kafkajs","message":"[BrokerPool] Failed to connect to seed broker, trying another broker from the list: Connection timeout","retryCount":5,"retryTime":7450} {"level":50,"time":"2023-08-31T15:17:48.213Z","pid":18,"hostname":"k8s-service-69975464ff-z2sgt","name":"svs-k8s-service","msg":"confluent kafka Kafka broker connnection failed. So, we cannot consume messages from confluent kafka Kafka Topic!!"} {"level":50,"time":"2023-08-31T15:17:48.213Z","pid":18,"hostname":"k8s-service-69975464ff-z2sgt","name":"svs-k8s-service","msg":"confluent kafka Kafka error {"name":"KafkaJSNumberOfRetriesExceeded","retriable":false,"retryCount":5,"retryTime":7450}"} └── / └── notifications/oapi (POST)

{"level":30,"time":"2023-08-31T15:17:48.227Z","pid":18,"hostname":"k8s-service-69975464ff-z2sgt","name":"svs-k8s-service","msg":"Server listening at http://0.0.0.0:8080"} {"level":"ERROR","timestamp":"2023-08-31T15:17:48.673Z","logger":"kafkajs","message":"[Connection] Connection error: read ECONNRESET","broker":"confluent-cloud-kafka:9092","clientId":"lloydsbank-group","stack":"Error: read ECONNRESET\n at TLSWrap.onStreamRead (node:internal/stream_base_commons:217:20)"}

nodejs kafkajs connection use http_proxy or https_proxy proxy

Setting HTTP_PROXY, HTTPS_PROXY and NO_PROXY doesn't seem to affect the node server's behavior, but I need a way to do this without having to manually edit libraries. I have ~10 libraries that reach out to external services that need to respect this proxying behavior. Is there some other way I can globally set proxying behavior including respecting NO_PROXY? — Hareesh Chatarasupalli, Sep 01 '23 at 08:44
Please clarify your specific problem or provide additional details to highlight exactly what you need. As it's currently written, it's hard to tell exactly what you're asking. — Community, Sep 03 '23 at 01:35

score 0 · Answer 1 · answered Sep 01 '23 at 08:56

'use strict';
const fp = require('fastify-plugin');
const { Kafka } = require('kafkajs');
const tls = require('tls')
const net = require('net')
const { createTunnel, closeTunnel } = require('proxy-chain');

async function KafkaPlugin(fastify, opts, done) {
    const kafka = new Kafka({
        clientId: fastify.config.KAFKA_CLIENT_ID,
        brokers: [fastify.config.KAFKA_BOOTSTRAP_SERVER],
        //  - SASL - tested on local dev-mac, using ssl true only getting msgs
        ssl: true,
        sasl: {
            mechanism: 'plain',
            username: fastify.config.KAFKA_CLIENT_SASL_API_KEY,
            password: fastify.config.KAFKA_CLIENT_SASL_API_SECRET
        },
        socketFactory: ({ host, port, ssl, onConnect }) => {
            const socket = ssl ? new tls.TLSSocket() : new net.Socket()

            createTunnel(process.env.HTTP_PROXY, `${host}:${port}`)
                .then((tunnelAddress) => {
                    // const [tunnelHost, tunnelPort] = tunnelAddress.split(':')
                    const tunnelHost = 'localhost';
                    const tunnelPort = tunnelAddress.split(':').slice('-1')[0];

                    socket.setKeepAlive(true, 60000)
                    socket.connect(
                        Object.assign({ host: tunnelHost, port: tunnelPort, servername: host }, ssl),
                        onConnect
                    )
                })
                .catch(error => socket.emit('error', error));

            return socket;
        }
    })
    const kafkaTopic = fastify.config.KAFKA_CONSUMER_TOPIC_NAME;

    try {
        const consumer = kafka.consumer({ groupId: fastify.config.KAFKA_CLIENT_ID });
        await consumer.connect();
        fastify.log.trace(' Kafka Borker connected successfully!!')

        await consumer.subscribe({ topic: kafkaTopic, fromBeginning: true });
        fastify.log.trace(` Kafka Topic - ${kafkaTopic} subscribed successfully!!`)

        //fastify.decorate('kafka_consumer', consumer);

        await consumer.run({
            autoCommit: false,
            eachMessage: async ({ topic, partition, message }) => {
                console.log({
                    topic,
                    partition,
                    offset: message.offset,
                    produced_datetime: new Date(message.timestamp / 1000).toISOString(),
                    value: message.value.toString(),

                });

                // Publish the  messages into LBG Kafka Topic
                await fastify.lbg_kafka_producer(message.value.toString());
            },
        });
    } catch (error) {
        fastify.log.error(' Kafka broker connnection failed. So, we cannot consume messages from  Kafka Topic!!');
        console.log(error);
    }

    done();
};
module.exports = fp(KafkaPlugin, { name: 'KafkaConsumer' });

As it’s currently written, your answer is unclear. Please [edit] to add additional details that will help others understand how this addresses the question asked. You can find more information on how to write good answers [in the help center](/help/how-to-answer). — Community, Sep 01 '23 at 18:02

nodejs application kafka communication over the http_proxy https_proxy proxy getting error

1 Answers1