0
Currently we are using CDP 7.1.7 and client wants to use Flume. Since CDP has removed Flume, we need to install it as a separate application. I have installed Flume at one of the data nodes. 
here are the config files:

flume-env.sh


    export JAVA_OPTS="$JAVA_OPTS -Djava.security.krb5.conf=/etc/krb5.conf
     -Djava.security.auth.login.config=/opt/cloudera/security/flafka_jaas.conf "

flafka_jaas.conf


    Client {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=true
      storeKey=true
      keyTab="flume.keytab" 
      principal="flume/hostname@realm";
    };
    
    KafkaClient {
      com.sun.security.auth.module.Krb5LoginModule required
      useKeyTab=true
      storeKey=true
      serviceName="kafka"
      keyTab="flume.keytab"
      principal="flume/hostname@realm";
    };

Flume.conf


    KafkaAgent.sources = source_kafka
       KafkaAgent.channels = MemChannel
       KafkaAgent.sinks = LoggerSink
    
       #Configuring Source
       KafkaAgent.sources.source_kafka.type = org.apache.flume.source.kafka.KafkaSource
       KafkaAgent.sources.source_kafka.kafka.bootstrap.servers = hostn1:9092,host2:9092,host3:9092
       KafkaAgent.sources.source_kafka.kafka.topics = cim
       KafkaAgent.sources.source_kafka.kafka.consumer.group.id = flume
        KafkaAgent.sources.source_kafka.channels = MemChannel
       #KafkaAgent.sources.source_kafka.kafka.consumer.timeout.ms = 100
       KafkaAgent.sources.source_kafka.agent-principal=flume/hostname@realm
       KafkaAgent.sources.source_kafka.agent-keytab=flume.keytab
       KafkaAgent.sources.source_kafka.kafka.consumer.security.protocol = SASL_PLAINTEXT
       KafkaAgent.sources.source_kafka.kafka.consumer.sasl.kerberos.service.name = kafka
        KafkaAgent.sources.source_kafka.kafka.consumer.sasl.mechanism = GSSAPI
       KafkaAgent.sources.source_kafka.kafka.consumer.security.protocol = SASL_PLAINTEXT
       #Configuring Sink
       KafkaAgent.sinks.LoggerSink.type = logger
    
       #Configuring Channel
       KafkaAgent.channels.MemChannel.type = memory
       KafkaAgent.channels.MemChannel.capacity = 10000
        KafkaAgent.channels.MemChannel.transactionCapacity = 1000
    
       #bind source and sink to channel
       KafkaAgent.sinks.LoggerSink.channel = MemChannel

After running this command:

`flume-ng agent -n KafkaAgent -c -conf /opt/cdpdeployment/apache-flume-1.9.0-bin/conf/ -f /opt/cdpdeployment/apache-flume-1.9.0-bin/conf/kafka-flume.conf  -Dflume.root.logger=DEBUG,console`

I am getting this below error:

    

Caused by: java.lang.IllegalArgumentException: Could not find a 'KafkaClient' entry in the JAAS configuration. System property 'java.security.auth.login.config' is not set
    at org.apache.kafka.common.security.JaasContext.defaultContext(JaasContext.java:133)
    at org.apache.kafka.common.security.JaasContext.load(JaasContext.java:98)
    at org.apache.kafka.common.security.JaasContext.loadClientContext(JaasContext.java:84)
    at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:119)
    at org.apache.kafka.common.network.ChannelBuilders.clientChannelBuilder(ChannelBuilders.java:65)
    at org.apache.kafka.clients.ClientUtils.createChannelBuilder(ClientUtils.java:88)
    at org.apache.kafka.clients.consumer.KafkaConsumer.<init>(KafkaConsumer.java:713)

Can someone tell me what I am missing as far as configs are concerned.

Advik
  • 1
  • 1

0 Answers0