0

i'm trying to deserialize an Avro message from kafka on apache flink

i'm currently doing it by implementing the DeserializationSchema interface but it is depecrated is there a better form to achieve this?

Thanks.

These are my classes

public final class FlinkKafkaExample {

public static void main(String[] args) throws Exception {

    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // setup Kafka sink
    ConfluentAvroDeserializationSchema deserSchema = new
            ConfluentAvroDeserializationSchema("http://localhost:8081", 1000);

    Properties kafkaProps = new Properties();
    kafkaProps.setProperty("bootstrap.servers", "0.0.0.0:9092");
    kafkaProps.setProperty("zookeeper.connect", "0.0.0.0:2181");
    kafkaProps.setProperty("group.id", "org.apache.flink");
    FlinkKafkaConsumer08<String> flinkKafkaConsumer =
            new FlinkKafkaConsumer08<String>("conekta.public.codes", deserSchema, kafkaProps);

    DataStream<String> kafkaStream = env.addSource(flinkKafkaConsumer);


    kafkaStream.print();
    env.execute("Flink Kafka Java Example");
}

}

public class ConfluentAvroDeserializationSchema implements DeserializationSchema<String> {

private final String schemaRegistryUrl;
private final int identityMapCapacity;
private KafkaAvroDecoder kafkaAvroDecoder;

public ConfluentAvroDeserializationSchema(String schemaRegistryUrl, int identityMapCapacity) {
    this.schemaRegistryUrl = schemaRegistryUrl;
    this.identityMapCapacity = identityMapCapacity;
}


public String deserialize(byte[] message) {
    if (kafkaAvroDecoder == null) {
        SchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient(this.schemaRegistryUrl, this.identityMapCapacity);
        this.kafkaAvroDecoder = new KafkaAvroDecoder(schemaRegistry);
    }
    return this.kafkaAvroDecoder.fromBytes(message).toString();
}


public boolean isEndOfStream(String nextElement) {
    return false;
}
public TypeInformation<String> getProducedType() {
    return BasicTypeInfo.STRING_TYPE_INFO;
}

}

OneCricketeer
  • 179,855
  • 19
  • 132
  • 245
whiteskull
  • 67
  • 9
  • If your data is Avro, then why are you saying that the data being deserizlied is a string? `DeserializationSchema` ... Also, `0.0.0.0:` is not a real address. Best to use `localhost`, where appropriate (and Flink should not need `zookeeper.connect` if you use `FlinkKafkaConsumer010` – OneCricketeer Jan 23 '20 at 23:26
  • You might want to use [ConfluentRegistryAvroDeserializationSchema](https://ci.apache.org/projects/flink/flink-docs-stable/api/java/org/apache/flink/formats/avro/registry/confluent/ConfluentRegistryAvroDeserializationSchema.html). See also https://stackoverflow.com/q/58849635/2000823. – David Anderson Jan 23 '20 at 19:47

0 Answers0