1

I have two almost similar kafka applications. They both listen to binlog for changes of two tables. My problem is one of them works fine but when try to launch the second one I receive the following exception.

org.apache.kafka.common.errors.SerializationException: Error registering Avro schema: {"type":"record","name":"Key","namespace":"mysql.company.payments","fields":[{"name":"id","type":"long"}],"connect.name":"mysql.company.payments.Key"} Caused by: io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException: Unexpected character ('<' (code 60)): expected a valid value (number, String, array, object, 'true', 'false' or 'null')

The schema file the error points to has the following content:

{
  "type": "record",
  "name": "Key",
  "namespace": "mysql.company.payments",
  "fields": [
    {
      "name": "id",
      "type": "long"
    }
  ],
  "connect.name": "mysql.company.payments.Key"
}

The other application that is working has the exact same avro file but the name of the table (payments) is replaced. Both apps are run from the same server and connected to the same Kafka cluster. I use maven plugin to create java classes based on avro files. The class Key.class gets created successfully.

These are the two important classes of my application:

Main class


import com.company.util.Configs;
import error.PaymentSerializationException;
import io.confluent.kafka.serializers.KafkaAvroSerializerConfig;
import io.confluent.kafka.streams.serdes.avro.GenericAvroSerde;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.StreamsConfig;
import payment.PaymentUpdateListener;

import java.util.Properties;

public class PaymentsMain {

    static Properties properties;

    public static void main(String[] args) {
        StreamsBuilder builder = new StreamsBuilder();
        properties = configProperties();

        StreamsBuilder streamsBuilder = watchForPaymentUpdate(builder);

        KafkaStreams kafkaStreams = new KafkaStreams(streamsBuilder.build(), properties);
        kafkaStreams.start();
        Runtime.getRuntime().addShutdownHook(new Thread(kafkaStreams::close));
    }

    private static StreamsBuilder watchForPaymentUpdate(StreamsBuilder builder){
        PaymentUpdateListener paymentUpdateListener = new PaymentUpdateListener(builder);
        paymentUpdateListener.start();
        return builder;
    }

    private static Properties configProperties(){

        Properties streamProperties = new Properties();

        streamProperties.put(KafkaAvroSerializerConfig.SCHEMA_REGISTRY_URL_CONFIG, Configs.getConfig("schemaRegistryUrl"));
        streamProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, "payment-kafka");
        streamProperties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, Configs.getConfig("bootstrapServerUrl"));
        streamProperties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
        streamProperties.put(StreamsConfig.STATE_DIR_CONFIG, "/tmp/state_dir");
        streamProperties.put(StreamsConfig.NUM_STREAM_THREADS_CONFIG, "3");
        streamProperties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, GenericAvroSerde.class);
        streamProperties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, GenericAvroSerde.class);
        streamProperties.put(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG, "DEBUG");
        streamProperties.put(StreamsConfig.DEFAULT_PRODUCTION_EXCEPTION_HANDLER_CLASS_CONFIG,
                PaymentSerializationException.class);

        return streamProperties;

    }
}

Stream class

import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.kstream.Consumed;

public class PaymentUpdateListener {

    private StreamsBuilder builder;

    public PaymentUpdateListener(StreamsBuilder builder) {
        this.builder = builder;
    }

    public void start(){

        builder.stream("mysql.company.payments",
                Consumed.with(PaymentSerde.getGenericKeySerde(), PaymentSerde.getEnvelopeSerde()))
                .to("kafka-consumer.payment");

    }
}
Community
  • 1
  • 1
Hessam
  • 1,377
  • 1
  • 23
  • 45

0 Answers0