We are facing this error in our Spring Cloud Gateway application deployed on Openshift, but it works well in the local but not in Openshift environment.
Below are the details.
Elastic APM Jar version: 1.30.0, 1.29.0 and 1.28.4
Spring boot version: spring-boot-2.3.10.RELEASE
OpenShift Master: v3.11.0+ec8630f-265
Kubernetes Master: v1.11.0+d4cacc0
OpenShift Web Console: v3.11.0+ea42280
Java: 11.0.14.1
JVM Args:
ERROR log:
Picked up JAVA_TOOL_OPTIONS: -XX:+UseContainerSupport
2022-03-30 14:19:31,658 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - Starting Elastic APM 1.30.0 as my-api-gateway on Java 11.0.14.1 Runtime version: 11.0.14.1+1 VM version: 11.0.14.1+1 (Eclipse Adoptium) Linux 3.10.0-1062.18.1.el7.x86_64
2022-03-30 14:19:31,662 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - service_name: my-api-gateway' (source: Environment Variables)
2022-03-30 14:19:31,662 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - environment: 'PR' (source: Environment Variables)
2022-03-30 14:19:31,662 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - enable_experimental_instrumentations: 'true' (source: Java System Properties)
2022-03-30 14:19:31,662 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - server_url: 'https://eapm-instance-nonprod.xxxxx.com:443' (source: Environment Variables)
2022-03-30 14:19:31,663 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - server_urls: 'https://eapm-instance-nonprod.xxxxx:443' (source: Environment Variables)
2022-03-30 14:19:31,663 [main] INFO co.elastic.apm.agent.configuration.StartupInfo - application_packages: 'com.my.api.gateway' (source: Environment Variables)
2022-03-30 14:19:33,463 [main] ERROR co.elastic.apm.agent.bci.IndyBootstrap - Provider "jar" not found
java.nio.file.ProviderNotFoundException: Provider "jar" not found
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at co.elastic.apm.agent.util.PackageScanner.getClassNames(PackageScanner.java:61) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.getClassNamesFromBundledPlugin(IndyBootstrap.java:440) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.bootstrap(IndyBootstrap.java:399) [elastic-apm-agent.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at java.lang.IndyBootstrapDispatcher.bootstrap(IndyBootstrapDispatcher.java:60) [?:?]
at java.lang.invoke.BootstrapMethodInvoker.invoke(Unknown Source) [?:?]
at java.lang.invoke.CallSite.makeSite(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSite(Unknown Source) [?:?]
at co.elastic.apm.agent.bci.bytebuddy.Instrumented.isInstrumented(Instrumented.java:34) [?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at co.elastic.apm.agent.bci.bytebuddy.InstallationListenerImpl.onAfterWarmUp(InstallationListenerImpl.java:65) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$InstallationListener$Compound.onAfterWarmUp(AgentBuilder.java:5221) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default$WarmupStrategy$Enabled.apply(AgentBuilder.java:11087) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default.doInstall(AgentBuilder.java:10774) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default.installOn(AgentBuilder.java:10712) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initInstrumentation(ElasticApmAgent.java:279) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initInstrumentation(ElasticApmAgent.java:164) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initialize(ElasticApmAgent.java:150) [elastic-apm-agent.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at co.elastic.apm.agent.premain.AgentMain.loadAndInitializeAgent(AgentMain.java:160) [elastic-apm-agent.jar:1.30.0]
at co.elastic.apm.agent.premain.AgentMain.init(AgentMain.java:101) [elastic-apm-agent.jar:1.30.0]
at co.elastic.apm.agent.premain.AgentMain.premain(AgentMain.java:50) [elastic-apm-agent.jar:1.30.0]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at sun.instrument.InstrumentationImpl.loadClassAndStartAgent(Unknown Source) [?:?]
at sun.instrument.InstrumentationImpl.loadClassAndCallPremain(Unknown Source) [?:?]
at org.springframework.boot.loader.Launcher.launch(Launcher.java:58) [gateway.jar:?]
at org.springframework.boot.loader.JarLauncher.main(JarLauncher.java:88) [gateway.jar:?]
#logback.classic pattern: %d %-5level [%thread] %logger{0}: %msg%n
. ____ _ __ _ _
/\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \
( ( )\___ | '_ | '_| | '_ \/ _` | \ \ \ \
\\/ ___)| |_)| | | | | || (_| | ) ) ) )
' |____| .__|_| |_|_| |_\__, | / / / /
=========|_|==============|___/=/_/_/_/
:: Spring Boot :: (v2.3.10.RELEASE)
2022-03-30 14:19:46,779 [main] ERROR co.elastic.apm.agent.bci.IndyBootstrap - Provider "jar" not found
java.nio.file.ProviderNotFoundException: Provider "jar" not found
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
2022-03-30 14:19:33,470 [main] ERROR co.elastic.apm.agent.bci.IndyBootstrap - Provider "jar" not found
java.nio.file.ProviderNotFoundException: Provider "jar" not found
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at co.elastic.apm.agent.util.PackageScanner.getClassNames(PackageScanner.java:61) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.getClassNamesFromBundledPlugin(IndyBootstrap.java:440) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.bootstrap(IndyBootstrap.java:399) [elastic-apm-agent.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at java.lang.IndyBootstrapDispatcher.bootstrap(IndyBootstrapDispatcher.java:60) [?:?]
at java.lang.invoke.BootstrapMethodInvoker.invoke(Unknown Source) [?:?]
at java.lang.invoke.CallSite.makeSite(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSite(Unknown Source) [?:?]
at co.elastic.apm.agent.bci.bytebuddy.Instrumented.isInstrumented(Instrumented.java:34) [?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at co.elastic.apm.agent.bci.bytebuddy.InstallationListenerImpl.onAfterWarmUp(InstallationListenerImpl.java:65) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$InstallationListener$Compound.onAfterWarmUp(AgentBuilder.java:5221) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default$WarmupStrategy$Enabled.apply(AgentBuilder.java:11087) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default.doInstall(AgentBuilder.java:10774) [elastic-apm-agent.jar:?]
at net.bytebuddy.agent.builder.AgentBuilder$Default.installOn(AgentBuilder.java:10712) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initInstrumentation(ElasticApmAgent.java:279) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initInstrumentation(ElasticApmAgent.java:164) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.ElasticApmAgent.initialize(ElasticApmAgent.java:150) [elastic-apm-agent.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at co.elastic.apm.agent.premain.AgentMain.loadAndInitializeAgent(AgentMain.java:160) [elastic-apm-agent.jar:1.30.0]
at co.elastic.apm.agent.premain.AgentMain.init(AgentMain.java:101) [elastic-apm-agent.jar:1.30.0]
at co.elastic.apm.agent.premain.AgentMain.premain(AgentMain.java:50) [elastic-apm-agent.jar:1.30.0]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at sun.instrument.InstrumentationImpl.loadClassAndStartAgent(Unknown Source) [?:?]
at sun.instrument.InstrumentationImpl.loadClassAndCallPremain(Unknown Source) [?:?]
2022-03-30 14:19:33,472 [main] WARN co.elastic.apm.agent.bci.bytebuddy.InstallationListenerImpl - Warmup: instrumented bytecode of co.elastic.apm.agent.bci.bytebuddy.Instrumented does not work as expected
2022-03-30 14:19:35,597 [main] INFO co.elastic.apm.agent.report.ReporterConfiguration - Both "server_urls" and "server_url" configuration options are set, therefore the "server_urls" configuration will be ignored.
2022-03-30 14:19:35,662 [elastic-apm-server-healthcheck] ERROR co.elastic.apm.agent.bci.IndyBootstrap - Provider "jar" not found
java.nio.file.ProviderNotFoundException: Provider "jar" not found
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at java.nio.file.FileSystems.newFileSystem(Unknown Source) ~[?:?]
at co.elastic.apm.agent.util.PackageScanner.getClassNames(PackageScanner.java:61) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.getClassNamesFromBundledPlugin(IndyBootstrap.java:440) ~[elastic-apm-agent.jar:?]
at co.elastic.apm.agent.bci.IndyBootstrap.bootstrap(IndyBootstrap.java:399) [elastic-apm-agent.jar:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) ~[?:?]
at java.lang.reflect.Method.invoke(Unknown Source) ~[?:?]
at java.lang.IndyBootstrapDispatcher.bootstrap(IndyBootstrapDispatcher.java:60) [?:?]
at java.lang.invoke.BootstrapMethodInvoker.invoke(Unknown Source) [?:?]
at java.lang.invoke.CallSite.makeSite(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSiteImpl(Unknown Source) [?:?]
at java.lang.invoke.MethodHandleNatives.linkCallSite(Unknown Source) [?:?]
at javax.net.ssl.SSLSocketFactory.getDefault(Unknown Source) [?:?]
at javax.net.ssl.HttpsURLConnection.getDefaultSSLSocketFactory(Unknown Source) [?:?]
at javax.net.ssl.HttpsURLConnection.<init>(Unknown Source) [?:?]
at sun.net.www.protocol.https.HttpsURLConnectionImpl.<init>(Unknown Source) [?:?]
at sun.net.www.protocol.https.Handler.openConnection(Unknown Source) [?:?]
at sun.net.www.protocol.https.Handler.openConnection(Unknown Source) [?:?]
at java.net.URL.openConnection(Unknown Source) [?:?]
at co.elastic.apm.agent.util.UrlConnectionUtils.openUrlConnectionThreadSafely(UrlConnectionUtils.java:29) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.report.ApmServerClient.startRequestToUrl(ApmServerClient.java:139) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.report.ApmServerClient.executeForAllUrls(ApmServerClient.java:276) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.report.ApmServerHealthChecker.call(ApmServerHealthChecker.java:65) [elastic-apm-agent.jar:?]
at co.elastic.apm.agent.report.ApmServerHealthChecker.call(ApmServerHealthChecker.java:42) [elastic-apm-agent.jar:?]
at java.util.concurrent.FutureTask.run(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
