0

My code consumes rocketmq information via flink, then stores data into hdfs, and when I run the program locally, everything works fine without any problems, but I get this error when I submit my flink job to the cluster。

SLF4J: Class path contains multiple SLF4J bindings.
SLF4J: Found binding in [jar:file:/home/work/ssd2/yarn/tjwqstaging-hdd/nodemanager/filecache/1287/log4j-slf4j-impl-2.17.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/work/ssd2/yarn/tjwqstaging-hdd/nodemanager/usercache/s_workspace_13975_krb/appcache/application_1688714553647_22917/filecache/21/0.0.17.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/work/packages/yarn/tjwqstaging-hdd/162b1cdda26b373716544e676f46318256027e86-20230711-204535/hadoop-3.1.0-mdh3.1.0.0-SNAPSHOT/share/hadoop/common/lib/slf4j-log4j12-1.7.25.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/work/packages/yarn/tjwqstaging-hdd/162b1cdda26b373716544e676f46318256027e86-20230711-204535/hadoop-3.1.0-mdh3.1.0.0-SNAPSHOT/share/hadoop/yarn/hadoop-yarn-autoscaler-3.1.0-mdh3.1.0.0-SNAPSHOT.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: Found binding in [jar:file:/home/work/packages/yarn/tjwqstaging-hdd/162b1cdda26b373716544e676f46318256027e86-20230711-204535/hadoop-3.1.0-mdh3.1.0.0-SNAPSHOT/share/hadoop/yarn/lib/log4j-slf4j-impl-2.15.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
SLF4J: Actual binding is of type [org.apache.logging.slf4j.Log4jLoggerFactory]
Exception in thread "main" java.lang.ExceptionInInitializerError
Caused by: java.lang.IllegalArgumentException: Cannot parse pattern '%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n'
    at org.apache.logging.log4j.core.layout.PatternLayout$SerializerBuilder.build(PatternLayout.java:490)
    at org.apache.logging.log4j.core.layout.PatternLayout.<init>(PatternLayout.java:140)
    at org.apache.logging.log4j.core.layout.PatternLayout.<init>(PatternLayout.java:61)
    at org.apache.logging.log4j.core.layout.PatternLayout$Builder.build(PatternLayout.java:770)
    at org.apache.logging.log4j.core.config.AbstractConfiguration.setToDefault(AbstractConfiguration.java:723)
    at org.apache.logging.log4j.core.config.DefaultConfiguration.<init>(DefaultConfiguration.java:47)
    at org.apache.logging.log4j.core.LoggerContext.<init>(LoggerContext.java:95)
    at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.createContext(ClassLoaderContextSelector.java:254)
    at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.locateContext(ClassLoaderContextSelector.java:218)
    at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:136)
    at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:123)
    at org.apache.logging.log4j.core.selector.ClassLoaderContextSelector.getContext(ClassLoaderContextSelector.java:117)
    at org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:150)
    at org.apache.logging.log4j.core.impl.Log4jContextFactory.getContext(Log4jContextFactory.java:47)
    at org.apache.logging.log4j.LogManager.getContext(LogManager.java:196)
    at org.apache.logging.log4j.spi.AbstractLoggerAdapter.getContext(AbstractLoggerAdapter.java:137)
    at org.apache.logging.slf4j.Log4jLoggerFactory.getContext(Log4jLoggerFactory.java:55)
    at org.apache.logging.log4j.spi.AbstractLoggerAdapter.getLogger(AbstractLoggerAdapter.java:47)
    at org.apache.logging.slf4j.Log4jLoggerFactory.getLogger(Log4jLoggerFactory.java:33)
    at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:329)
    at org.slf4j.LoggerFactory.getLogger(LoggerFactory.java:349)
    at org.apache.flink.runtime.entrypoint.ClusterEntrypoint.<clinit>(ClusterEntrypoint.java:110)
Caused by: java.lang.SecurityException: Invalid signature file digest for Manifest main attributes
    at sun.security.util.SignatureFileVerifier.processImpl(SignatureFileVerifier.java:330)
    at sun.security.util.SignatureFileVerifier.process(SignatureFileVerifier.java:263)
    at java.util.jar.JarVerifier.processEntry(JarVerifier.java:318)
    at java.util.jar.JarVerifier.update(JarVerifier.java:230)
    at java.util.jar.JarFile.initializeVerifier(JarFile.java:383)
    at java.util.jar.JarFile.getInputStream(JarFile.java:455)
    at sun.net.www.protocol.jar.JarURLConnection.getInputStream(JarURLConnection.java:164)
    at java.net.URL.openStream(URL.java:1068)
    at org.apache.logging.log4j.core.config.plugins.processor.PluginCache.loadCacheFiles(PluginCache.java:97)
    at org.apache.logging.log4j.core.config.plugins.util.PluginRegistry.decodeCacheFiles(PluginRegistry.java:166)
    at org.apache.logging.log4j.core.config.plugins.util.PluginRegistry.loadFromMainClassLoader(PluginRegistry.java:119)
    at org.apache.logging.log4j.core.config.plugins.util.PluginManager.collectPlugins(PluginManager.java:132)
    at org.apache.logging.log4j.core.pattern.PatternParser.<init>(PatternParser.java:132)
    at org.apache.logging.log4j.core.pattern.PatternParser.<init>(PatternParser.java:113)
    at org.apache.logging.log4j.core.layout.PatternLayout.createPatternParser(PatternLayout.java:255)
    at org.apache.logging.log4j.core.layout.PatternLayout$SerializerBuilder.build(PatternLayout.java:473)
    ... 21 more

I modified the flink rocketmq connector that the pom file depends on, but I didn't change any log dependencies

this my pom

 <dependency>
      <groupId>org.scala-lang</groupId>
      <artifactId>scala-library</artifactId>
      <version>2.12.4</version>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-java</artifactId>
      <version>${flink.version}</version>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-api</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-shaded-netty</artifactId>
      <version>4.1.70.Final-15.0</version>
    </dependency>


    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-java_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>
     <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-base</artifactId>
      <version>${flink.version}</version>

    </dependency>
  <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-files</artifactId>
      <version>${flink.version}</version>

    </dependency>



    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-parquet_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>




    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-common</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-clients_2.12</artifactId>
      <version>${flink.version}</version>
    </dependency>
 <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-api-java-bridge_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-table-runtime_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-queryable-state-runtime</artifactId>
      <version>${flink.version}</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-avro</artifactId>
      <version>${flink.version}</version>

    </dependency>
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-runtime-web_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <groupId>org.apache.flink</groupId>
          <artifactId>flink-clients</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

 <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-rocketmq</artifactId>
      <version>1.14.1</version>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-api</artifactId>
        </exclusion>
        <exclusion>
          <groupId>ch.qos.logback</groupId>
          <artifactId>logback-classic</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
 <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-hadoop-compatibility_2.12</artifactId>
      <version>${flink.version}</version>
      <scope>test</scope>
    </dependency>


    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->


    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>3.1.0-mdh3.1.1.23</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs-client</artifactId>
      <version>3.1.0-mdh3.1.1.23</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>3.1.0-mdh3.1.1.23</version>
      <exclusions>
        <exclusion>
          <groupId>org.slf4j</groupId>
          <artifactId>slf4j-log4j12</artifactId>
        </exclusion>
      </exclusions>
    </dependency>
 <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-hadoop</artifactId>
      <version>1.12.0</version>
    </dependency>

    
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.12</artifactId>
      <version>${spark.version}</version>
    </dependency>

    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.12</artifactId>
      <version>${spark.version}</version>
    </dependency>
 <dependency>
      <groupId>org.apache.parquet</groupId>
      <artifactId>parquet-avro</artifactId>
      <version>1.12.2</version>
      <optional>true</optional>
      <exclusions>
        <exclusion>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-client</artifactId>
        </exclusion>
        <exclusion>
          <groupId>it.unimi.dsi</groupId>
          <artifactId>fastutil</artifactId>
        </exclusion>
      </exclusions>
    </dependency>

I tried changing the config file of log4j.properties but it didn't work, and the format in the error I didn't use and I didn't find him anywhere, I want to know how to fix this.

this is my log4j (update or not)

\#log4j.rootLogger=INFO, dledger  
\#log4j.appender.stdout=org.apache.log4j.ConsoleAppender  
\#log4j.appender.stdout.Target=System.out  
\#log4j.appender.stdout.layout=org.apache.log4j.PatternLayout  
\#log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c{1}:%L - %m%n  
\#log4j.appender.dledger=org.apache.log4j.RollingFileAppender  
\#log4j.appender.dledger.File=${user.home}/logs/dledger/dledger.log  
\#log4j.appender.dledger.Append=true  
\#log4j.appender.dledger.MaxFileSize=500000000  
\#log4j.appender.dledger.MaxBackupIndex=10  
\#log4j.appender.dledger.layout=org.apache.log4j.PatternLayout  
\#log4j.appender.dledger.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %c{1}:%L - %m%n  
\#log4j.logger.RocketmqRemoting=ERROR, dledger  
\#log4j.additivity.RocketmqRemoting=false  
\#log4j.logger.io.openmessaging.storage.dledger.cmdline=INFO, stdout  
\#log4j.additivity.io.openmessaging.storage.dledger.cmdline=false  
\#log4j.rootLogger = info,stdout  
\#log4j.appender.stdout = org.apache.log4j.ConsoleAppender  
\#log4j.appender.stdout.Target = System.out  
\#log4j.appender.stdout.layout = org.apache.log4j.PatternLayout  
\#log4j.appender.stdout.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss,SSS} \[%-5p\] method:%l%n%m%n  
log4j.rootLogger=INFO,console

\#??????????  
log4j.appender.console = org.apache.log4j.ConsoleAppender  
log4j.appender.console.Target = System.out  
log4j.appender.console.Threshold=DEBUG  
log4j.appender.console.layout = org.apache.log4j.PatternLayout  
log4j.appender.console.layout.ConversionPattern=\[%c\]-%m%n

0 Answers0