0

I have an AWS MSK cluster that is using IAM authentication.I am trying to connect with my node app, but I get an error in the consumer part of Kafka. my producer code is :

const { Kafka,logLevel } = require('kafkajs');

const kafka = new Kafka({
    clientId: 'api_gateway_client',
    brokers: ["b-1.ashique.exxxx.xxxxafka.ap-southexxxx.xxzonaws.com:9098"],
    ssl: true,
 //  sasl: {
//    mechanism: 'aws',
//    authorizationIdentity: '882xxxx397', // UserId or RoleId
//    accessKeyId: 'AKIAxxxxxD37NEJ',
//    secretAccessKey: 'q7SI3JxxxxxxxMhY0ciAou6TDXWdyR6h',
    //sessionToken: 'WHArYt8i5vfxxxxxxv/Eww6eL9tgQMJp6QFNEXAMPLETOKEN' // Optional
//  },
})
const producer = kafka.producer();
producer.connect();

export async function sendKafkaMessage(topic: string, message: string): Promise<any> {

    const result = await producer.send({
        topic,
        messages: [
            { value: message }
        ]
    });
    return result;
}

producer.on('producer.connect',() => {
    console.log('Kafka producer connected ................');
    const admin = kafka.admin();



});

my consumer code is:

//import { Kafka, Consumer, EachMessagePayload } from 'kafkajs'
const { Kafka,logLevel } = require('kafkajs');


//code for plain mechanism
 const kafka = new Kafka({
     clientId: 'api_gateway_client',
     brokers: ["b-1.ashique.et7xxxxx.xxx.xxxonaws.com:9098"],
     ssl: true,
//     logLevel: logLevel.DEBUG,
//     sasl: {
//         mechanism: 'plain',
//         username: "AKIxxxxxD37NEJ",
//         password: "q7SI3JyxxxxxxciAou6TDXWdyR6h"
//    }
// });
});


const admin = kafka.admin();

async function listTopics() {
  await admin.connect();
  const topics = await admin.listTopics();

  console.log('Topics:', topics);
}

type Ttopic = {
    Name: string,
    FromBeginning: boolean,

}

const topics: Ttopic[] = [
    { Name: 'ashique',FromBeginning: false } // first create a topic in kafka and specify the topic here
]

// Define the consumer options
const consumerOptions = {
    groupId: 'api_gateway_group',
    maxWaitTimeInMs: 5000,
}

// Define the function that will handle incoming messages
async function handleMessage({ topic, partition, message }: any) {
    console.log(`Received message on topic ${topic}, partition ${partition}: ${message.value.toString()}`)
}

// Create the KafkaJS consumer instance
const consumer: any = kafka.consumer(consumerOptions)

async function startConsumer() {
    // Connect to the Kafka brokers
    await consumer.connect()

    // Subscribe to the topics
    for (const topic of topics) {
        console.log("topicName ==",topic.Name);
        await consumer.subscribe({
            topic: topic.Name,
            fromBeginning: topic.FromBeginning
        });
    }

    await consumer.run({
        eachMessage: handleMessage,
    });
}
listTopics().catch(error => {
  console.error('Error listing topics:', error);
})
// Start the consumer
startConsumer().catch(error => {
  console.log('Error starting consumer::::', error);
});

When i hit npm start, i get the error as "

{"level":"WARN","timestamp":"2023-08-26T06:00:45.361Z","logger":"kafkajs","message":"KafkaJS v2.0.0 switched default partitioner. To retain the same partitioning behavior as in previous versions, create the producer with the option \"createPartitioner: Partitioners.LegacyPartitioner\". See the migration guide at https://kafka.js.org/docs/migration-guide-v2.0.0#producer-new-default-partitioner for details. Silence this warning by setting the environment variable \"KAFKAJS_NO_PARTITIONER_WARNING=1\""}
port running on - 5000
Kafka producer connected ................
topicName == ashique
Error listing topics: KafkaJSConnectionClosedError: Closed connection
    at TLSSocket.onEnd (/var/www/html/KafkaTestCode/node_modules/kafkajs/src/network/connection.js:197:13)
    at TLSSocket.emit (node:events:525:35)
    at TLSSocket.emit (node:domain:489:12)
    at endReadableNT (node:internal/streams/readable:1359:12)
    at processTicksAndRejections (node:internal/process/task_queues:82:21) {
  retriable: true,
  helpUrl: undefined,
  broker: 'b-1.ashiquxxxxx.xxxx.xxxxxxazonaws.com:9098',
  code: undefined,
  host: 'b-1.ashiquxxxxx.xxxx.xxxxxxazonaws.com',
  port: 9098,
  [cause]: undefined
}
Error starting consumer:::: KafkaJSConnectionClosedError: Closed connection
    at TLSSocket.onEnd (/var/www/html/KafkaTestCode/node_modules/kafkajs/src/network/connection.js:197:13)
    at TLSSocket.emit (node:events:525:35)
    at TLSSocket.emit (node:domain:489:12)
    at endReadableNT (node:internal/streams/readable:1359:12)
    at processTicksAndRejections (node:internal/process/task_queues:82:21) {
  retriable: true,
  helpUrl: undefined,
  broker: 'b-1.ashiquxxxxx.xxxx.xxxxxxazonaws.com:9098',
  code: undefined,
  host: 'b-1.ashiquxxxxx.xxxx.xxxxxxazonaws.com',
  port: 9098,
  [cause]: undefined
}

the node is in the same vpc of MSK cluster.Pls not that i ve temp commented out sasl in the producer and consumer code just for testing purpose

OneCricketeer
  • 179,855
  • 19
  • 132
  • 245
ashique
  • 935
  • 2
  • 8
  • 26
  • Did the documentation not work for you? Because your code isn't how you use IAM https://kafka.js.org/docs/configuration#aws-iam-example – OneCricketeer Aug 26 '23 at 12:32
  • yeah i ve added iam sasl part.but now i am getting "[Connection] Connection timeout","broker":"b-2-public.xxxxxxx.xxxxoutheast-1.amazonaws.com:9092","clientId":"api_gateway_client"}". telnet to the broker with port 9092 is not accessible.but telnet with 9198 is accessible. – ashique Aug 30 '23 at 06:58
  • Unclear why you're trying to use 9092 when that's not the correct port for IAM – OneCricketeer Aug 30 '23 at 12:09
  • when telnet to broker:9092 , i am getting the connection.but when i am using the boker and 9092 in my nodejs app, it show the error "type: 'UNSUPPORTED_SASL_MECHANISM'" .i have used iam sasl type correctly – ashique Aug 30 '23 at 12:38
  • You keep saying telnet, but that's not how you test authentication protocols. 9092 **is not** the SASL or IAM port. Look at the docs I've shared three times now – OneCricketeer Aug 30 '23 at 12:41

1 Answers1

0

You need to use different ports. 9098 requires IAM

https://docs.aws.amazon.com/msk/latest/developerguide/port-info.html

You've only shown a consumer error, not a producer one, so I assume that worked fine and you should create only one Kafka object / config and import across both files rather than define different connection types

OneCricketeer
  • 179,855
  • 19
  • 132
  • 245