ClassNotFoundException while running jar on Amazon Kinesis Streaming Analytics app

Question

I have created a Kinesis Analytics Streaming Application in SpringBoot which will consume messages from the AmazonKinesis input stream and will do some operations on top of it using the Apache Flink DataStream library.

When, I am uploading the application jar to S3 and trying to run this application on Streaming App it is throwing ClassNotFoundException for one of the files which is the ApplicationConfiguration file. Also, when I am running this application locally, it is running fine without any errors and I am able to consume the messages.

Below are some of the code files.

ApplicationConfiguration.java

import java.util.Properties;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer;
import org.apache.flink.streaming.connectors.kinesis.FlinkKinesisProducer;
import org.apache.flink.streaming.connectors.kinesis.config.ConsumerConfigConstants;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

@Configuration
public class ApplicationConfiguration {

  @Value("${oe.metric.input.stream}")
  private String inputStreamName;
  @Value("${oe.metric.output.stream}")
  private String outputStreamName;
  @Value("${aws.region}")
  private String region;
  @Value("${stream.initial.position}")
  private String position;

  @Bean
  public FlinkKinesisProducer<String> createSinkFromStaticConfig() {
    Properties outputProperties = new Properties();
    outputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, region);
    outputProperties.setProperty("AggregationEnabled", "false");

    FlinkKinesisProducer<String> sink = new FlinkKinesisProducer<>(new SimpleStringSchema(),
        outputProperties);
    sink.setDefaultStream(outputStreamName);
    sink.setDefaultPartition("0");
    return sink;
  }

  @Bean
  public DataStream<String> createSourceFromConfig() {
    Properties inputProperties = new Properties();
    inputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, region);
    inputProperties.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, position);

    return ApplicationConstants.streamExecutionEnvironment.addSource(
        new FlinkKinesisConsumer<>(inputStreamName, new SimpleStringSchema(), inputProperties));
  }

Error:

Caused by: java.lang.IllegalStateException: Cannot load configuration class: com.pkg.config.ApplicationConfiguration
at org.springframework.context.annotation.ConfigurationClassPostProcessor.enhanceConfigurationClasses(ConfigurationClassPostProcessor.java:415)
at org.springframework.context.annotation.ConfigurationClassPostProcessor.postProcessBeanFactory(ConfigurationClassPostProcessor.java:268)
at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:325)
at org.springframework.context.support.PostProcessorRegistrationDelegate.invokeBeanFactoryPostProcessors(PostProcessorRegistrationDelegate.java:147)
at org.springframework.context.support.AbstractApplicationContext.invokeBeanFactoryPostProcessors(AbstractApplicationContext.java:746)
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:564)
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:734)
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:408)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:308)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1306)
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1295)
at com.pkg.KinesisConsumerApplication.main(KinesisConsumerApplication.java:18)
... 25 more
Caused by: java.lang.ClassNotFoundException: com.pkg.config.ApplicationConfiguration
at java.base/java.net.URLClassLoader.findClass(URLClassLoader.java:476)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:589)
at org.springframework.boot.loader.LaunchedURLClassLoader.loadClass(LaunchedURLClassLoader.java:151)
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:522)
at java.base/java.lang.Class.forName0(Native Method)
at java.base/java.lang.Class.forName(Class.java:398)

KinesisConsumerApplication.java

@SpringBootApplication
@EnableAutoConfiguration
@ComponentScan(basePackages = "com.pkg")
public class KinesisConsumerApplication implements CommandLineRunner {

  @Autowired
  ApplicationContext ctx;

  public static void main(String[] args) {
    SpringApplication.run(KinesisConsumerApplication.class, args);
  }

  @Override
  public void run(String... args) throws Exception {
    GetKinesisRecords getKinesisRecords = ctx.getBean(GetKinesisRecords.class);
    getKinesisRecords.getDataFromStream();
  }
}

Maven pom.xml file

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xmlns="http://maven.apache.org/POM/4.0.0"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
  <artifactId>kinesis-consumer</artifactId>
  <build>
    <plugins>
      <plugin>
        <artifactId>spring-boot-maven-plugin</artifactId>
        <configuration>
          <excludes>
            <exclude>
              <artifactId>lombok</artifactId>
              <groupId>org.projectlombok</groupId>
            </exclude>
          </excludes>
        </configuration>
        <groupId>org.springframework.boot</groupId>
      </plugin>
    </plugins>
  </build>
  <dependencies>

    <dependency>
      <artifactId>spring-boot-starter</artifactId>
      <groupId>org.springframework.boot</groupId>
      <exclusions>
        <exclusion>
          <groupId>org.springframework.boot</groupId>
          <artifactId>spring-boot-starter-logging</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

    <dependency>
      <artifactId>flink-connector-kinesis_${scala.binary.version}</artifactId>
      <groupId>org.apache.flink</groupId>
      <version>1.13.2</version>
    </dependency>

    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
      <version>1.13.2</version>
      <scope>compile</scope>
    </dependency>

    <dependency>
      <artifactId>flink-core</artifactId>
      <groupId>org.apache.flink</groupId>
      <version>1.13.2</version>
    </dependency>

    <dependency>
      <artifactId>jackson-datatype-jsr310</artifactId>
      <groupId>com.fasterxml.jackson.datatype</groupId>
    </dependency>

    <dependency>
      <artifactId>spring-boot-starter-test</artifactId>
      <groupId>org.springframework.boot</groupId>
      <scope>test</scope>
    </dependency>

    <dependency>
      <artifactId>junit</artifactId>
      <groupId>junit</groupId>
      <scope>test</scope>
      </dependency>

    <dependency>
      <artifactId>ssai-common</artifactId>
      <groupId>com.adsparx.phoenix</groupId>
      <version>1.2.0</version>
    </dependency>

    <dependency>
      <artifactId>flink-clients</artifactId>
      <groupId>org.apache.flink</groupId>
      <version>1.15.0</version>
    </dependency>

    <dependency>
      <artifactId>jakarta.xml.bind-api</artifactId>
      <groupId>jakarta.xml.bind</groupId>
    </dependency>

    <!-- Logging -->
    <dependency>
      <groupId>org.apache.logging.log4j</groupId>
      <artifactId>log4j-slf4j-impl</artifactId>
    </dependency>

  </dependencies>

  <description>Apache Kinesis Consumer</description>
  <groupId>com.pkg</groupId>
  <modelVersion>4.0.0</modelVersion>
  <name>kinesis-consumer</name>
  <parent>
    <artifactId>spring-boot-starter-parent</artifactId>
    <groupId>org.springframework.boot</groupId>
    <relativePath/>
    <version>2.4.0</version>
  </parent>
  <properties>
    <java.version>11</java.version>
    <scala.binary.version>2.12</scala.binary.version>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
  </properties>

  <version>0.0.1-SNAPSHOT</version>

</project>

Things that I have tried on my end:

Clean build the project a couple of times.
Checked Java and Spring version in pom, as suggested in one of the StackOverflow post.
Downgraded SpringBoot version from "2.7.0" to "2.4.0".
Added ComponentScan annotation in main class.
Tried changing the annotation from @Configuration to @Component.
Last but not least, tried google and stackoverflow.

Can someone please help me identify the error?

score 0 · Answer 1 · answered Aug 19 '22 at 04:18

0

Is this because you're attempting to use dependencies for flink 1.15? AFAIK, 1.13 is the latest version supported. You need to choose which flink version you intend to use when creating the streaming application.

answered Aug 19 '22 at 04:18

Jordan Morris

2,101
2
24
41

ClassNotFoundException while running jar on Amazon Kinesis Streaming Analytics app

1 Answers1