I have a springboot application that consume a xml and its running inside a docker container. When I do a load testing for that at some point its throwing "Broken pipe" error. I'm increasing traffic by 20% continuously with 5min gap (100% traffic is around 300 requests per 10 seconds). Used the goreplay tool for the traffic sending. From the begining also time to time this error throw and its less count (around 4 to 8 per 10 seconds but not continuously). But around 280% traffic level this error count is very high and continuously occuring.
Can please anyone let me know what can be the reasons for this and how to resolve this? Or what area I need to check to fix this. With the monitoring graphs I couldn't see any server matrics limit exceed or any other errors in the logs. Getting the error as below.
Thank you.
[org.apache.catalina.connector.ClientAbortException: java.io.IOException: Broken pipe]] with root cause
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?]
at sun.nio.ch.SocketDispatcher.write(Unknown Source) ~[?:?]
at sun.nio.ch.IOUtil.writeFromNativeBuffer(Unknown Source) ~[?:?]
at sun.nio.ch.IOUtil.write(Unknown Source) ~[?:?]
at sun.nio.ch.IOUtil.write(Unknown Source) ~[?:?]
at sun.nio.ch.SocketChannelImpl.write(Unknown Source) ~[?:?]
at org.apache.tomcat.util.net.NioChannel.write(NioChannel.java:135) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.tomcat.util.net.NioBlockingSelector.write(NioBlockingSelector.java:118) ~[tomcat-embed-core-9.0.44.jar!/:9.0.4
4]
at org.apache.tomcat.util.net.NioSelectorPool.write(NioSelectorPool.java:151) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.tomcat.util.net.NioEndpoint$NioSocketWrapper.doWrite(NioEndpoint.java:1367) ~[tomcat-embed-core-9.0.44.jar!/:9
.0.44]
at org.apache.tomcat.util.net.SocketWrapperBase.doWrite(SocketWrapperBase.java:766) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.tomcat.util.net.SocketWrapperBase.writeBlocking(SocketWrapperBase.java:586) ~[tomcat-embed-core-9.0.44.jar!/:9
.0.44]
at org.apache.tomcat.util.net.SocketWrapperBase.write(SocketWrapperBase.java:530) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.coyote.http11.Http11OutputBuffer$SocketOutputBuffer.doWrite(Http11OutputBuffer.java:546) ~[tomcat-embed-core-9
.0.44.jar!/:9.0.44]
at org.apache.coyote.http11.filters.ChunkedOutputFilter.doWrite(ChunkedOutputFilter.java:112) ~[tomcat-embed-core-9.0.44.jar
!/:9.0.44]
at org.apache.coyote.http11.Http11OutputBuffer.doWrite(Http11OutputBuffer.java:193) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.coyote.Response.doWrite(Response.java:606) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.OutputBuffer.realWriteBytes(OutputBuffer.java:340) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.OutputBuffer.flushByteBuffer(OutputBuffer.java:783) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44
]
at org.apache.catalina.connector.OutputBuffer.append(OutputBuffer.java:688) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.OutputBuffer.writeBytes(OutputBuffer.java:388) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.OutputBuffer.write(OutputBuffer.java:366) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.CoyoteOutputStream.write(CoyoteOutputStream.java:96) ~[tomcat-embed-core-9.0.44.jar!/:9.0.4
4]
at com.sun.xml.bind.v2.runtime.output.UTF8XmlOutput.write(UTF8XmlOutput.java:381) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.output.Encoded.write(Encoded.java:137) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.output.UTF8XmlOutput.attribute(UTF8XmlOutput.java:224) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.attribute(XMLSerializer.java:416) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.AttributeProperty.serializeAttributes(AttributeProperty.java:73) ~[jaxb-runtime-2.3.
3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeAttributes(ClassBeanInfoImpl.java:352) ~[jaxb-runtime-2.3.3.jar!/:
2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:659) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.serializeItem(ArrayElementNodeProperty.java:39) ~[jaxb-runt
ime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementProperty.serializeListBody(ArrayElementProperty.java:142) ~[jaxb-runtime
-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayERProperty.serializeBody(ArrayERProperty.java:129) ~[jaxb-runtime-2.3.3.jar!/:2
.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:666) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.SingleElementNodeProperty.serializeBody(SingleElementNodeProperty.java:128) ~[jaxb-r
untime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:666) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.SingleElementNodeProperty.serializeBody(SingleElementNodeProperty.java:128) ~[jaxb-r
untime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:666) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.serializeItem(ArrayElementNodeProperty.java:39) ~[jaxb-runt
ime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementProperty.serializeListBody(ArrayElementProperty.java:142) ~[jaxb-runtime
-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayERProperty.serializeBody(ArrayERProperty.java:129) ~[jaxb-runtime-2.3.3.jar!/:2
.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:666) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementNodeProperty.serializeItem(ArrayElementNodeProperty.java:39) ~[jaxb-runt
ime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayElementProperty.serializeListBody(ArrayElementProperty.java:142) ~[jaxb-runtime
-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.ArrayERProperty.serializeBody(ArrayERProperty.java:129) ~[jaxb-runtime-2.3.3.jar!/:2
.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsXsiType(XMLSerializer.java:666) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.property.SingleElementNodeProperty.serializeBody(SingleElementNodeProperty.java:128) ~[jaxb-r
untime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeBody(ClassBeanInfoImpl.java:329) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsSoleContent(XMLSerializer.java:563) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.ClassBeanInfoImpl.serializeRoot(ClassBeanInfoImpl.java:310) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.XMLSerializer.childAsRoot(XMLSerializer.java:464) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.MarshallerImpl.write(MarshallerImpl.java:298) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at com.sun.xml.bind.v2.runtime.MarshallerImpl.marshal(MarshallerImpl.java:226) ~[jaxb-runtime-2.3.3.jar!/:2.3.3]
at org.springframework.http.converter.xml.Jaxb2RootElementHttpMessageConverter.writeToResult(Jaxb2RootElementHttpMessageConv
erter.java:183) ~[spring-web-5.3.5.jar!/:5.3.5]
at org.springframework.http.converter.xml.AbstractXmlHttpMessageConverter.writeInternal(AbstractXmlHttpMessageConverter.java
:85) ~[spring-web-5.3.5.jar!/:5.3.5]
at org.springframework.http.converter.AbstractHttpMessageConverter.write(AbstractHttpMessageConverter.java:227) ~[spring-web
-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.annotation.AbstractMessageConverterMethodProcessor.writeWithMessageConverters(
AbstractMessageConverterMethodProcessor.java:293) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.annotation.RequestResponseBodyMethodProcessor.handleReturnValue(RequestRespons
eBodyMethodProcessor.java:181) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.method.support.HandlerMethodReturnValueHandlerComposite.handleReturnValue(HandlerMethodReturnValu
eHandlerComposite.java:78) ~[spring-web-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandl
erMethod.java:124) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHand
lerAdapter.java:894) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAd
apter.java:808) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[sp
ring-webmvc-5.3.5.jar!/:5.3.5]
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1060) ~[spring-webmvc-5.3.5.jar!/:5.3
.5]
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:962) ~[spring-webmvc-5.3.5.jar!/:5.3.5
]
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.3.5.jar!/:5
.3.5]
at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:652) ~[tomcat-embed-core-9.0.44.jar!/:4.0.FR]
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.3.5.jar!/:5.3.5]
at javax.servlet.http.HttpServlet.service(HttpServlet.java:733) ~[tomcat-embed-core-9.0.44.jar!/:4.0.FR]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) ~[tomcat-embed-websocket-9.0.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) ~[spring-web-5.3.5.ja
r!/:5.3.5]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.3.5.jar!/:5.3.
5]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93) ~[spring-web-5.3.5.jar!/:5.3
.5]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.3.5.jar!/:5.3.
5]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at datadog.trace.instrumentation.springweb.HandlerMappingResourceNameFilter.doFilterInternal(HandlerMappingResourceNameFilte
r.java:49) ~[?:?]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.3.5.jar!/:5.3.
5]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:93) ~[
spring-boot-actuator-2.4.4.jar!/:2.4.4]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.3.5.jar!/:5.3.
5]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) ~[spring-web-5.
3.5.jar!/:5.3.5]
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:119) ~[spring-web-5.3.5.jar!/:5.3.
5]
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0
.44.jar!/:9.0.44]
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.44.jar!
/:9.0.44]
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:202) ~[tomcat-embed-core-9.0.44.jar!/:9.0.
44]
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97) ~[tomcat-embed-core-9.0.44.jar!/:9.0.4
4]
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:542) ~[tomcat-embed-core-9.0.44.jar!/:9
.0.44]
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:143) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:357) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:374) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:893) ~[tomcat-embed-core-9.0.44.jar!/:
9.0.44]
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1707) ~[tomcat-embed-core-9.0.44.jar!/:9.0.
44]
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) ~[tomcat-embed-core-9.0.44.jar!/:9.0.44]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) ~[?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) ~[?:?]
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) ~[tomcat-embed-core-9.0.44.jar!/:9.0.4
4]
