3

AKS Kubernetes version 1.20.7 ES Version: 7.12.0

  • We are getting 'Empty client certificate chain' (see log trace below)
  • We enabled TLS (setting below)
  • We are using self signed certificates, not using elasticsearch-certutil tool
  • Out certs have 'Extended Key Usage' that explicitly enable both clientAuth and serverAuth

Any help is really appreciated

XPack Setting

xpack.security.enabled: true
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: /usr/share/Elasticsearch/config/certs/elasticsearch.pfx
xpack.security.transport.ssl.truststore.path: /usr/share/Elasticsearch/config/certs/elasticsearch.pfx
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.client_authentication: optional
xpack.security.http.ssl.verification_mode: certificate
xpack.security.http.ssl.keystore.path: /usr/share/Elasticsearch/config/certs/elasticsearch.pfx
xpack.security.http.ssl.truststore.path: /usr/share/Elasticsearch/config/certs/elasticsearch.pfx

Log Trace

ompletionEvent(javax.net.ssl.SSLHandshakeException: Empty client certificate chain)" }
{"type": "server", "timestamp": "2021-10-27T05:32:39,760Z", "level": "TRACE", "component": "o.e.t.n.ESLoggingHandler", "cluster.name": "test-rp-search", "node.name": "test-rp-search-master-0", "message": "[id: 0x7272c902, L:/10.155.12.55:9300 ! R:/10.155.12.34:42246] EXCEPTION: io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Empty client certificate chain",
"stack trace": ["io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Empty client certificate chain",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:471) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at java.lang.Thread.run(Thread.java:832) [?:?]",
"Caused by: javax.net.ssl.SSLHandshakeException: Empty client certificate chain",
"at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?]",
"at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:356) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:312) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:303) ~[?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.onConsumeCertificate(CertificateMessage.java:1194) ~[?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.consume(CertificateMessage.java:1181) ~[?:?]",
"at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?]",
"at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1267) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1254) ~[?:?]",
"at java.security.AccessController.doPrivileged(AccessController.java:691) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1199) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.runAllDelegatedTasks(SslHandler.java:1542) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1556) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1440) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1267) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1314) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:501) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"... 16 more"] }
{"type": "server", "timestamp": "2021-10-27T05:32:39,762Z", "level": "WARN", "component": "o.e.t.TcpTransport", "cluster.name": "test-rp-search", "node.name": "test-rp-search-master-0", "message": "exception caught on transport layer [Netty4TcpChannel{localAddress=/10.155.12.55:9300, remoteAddress=/10.155.12.34:42246, profile=default}], closing connection",
"stacktrace": ["io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Empty client certificate chain",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:471) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at java.lang.Thread.run(Thread.java:832) [?:?]",
"Caused by: javax.net.ssl.SSLHandshakeException: Empty client certificate chain",
"at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?]",
"at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:356) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:312) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:303) ~[?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.onConsumeCertificate(CertificateMessage.java:1194) ~[?:?]",
"at sun.security.ssl.CertificateMessage$T13CertificateConsumer.consume(CertificateMessage.java:1181) ~[?:?]",
"at sun.security.ssl.SSLHandshake.consume(SSLHandshake.java:396) ~[?:?]",
"at sun.security.ssl.HandshakeContext.dispatch(HandshakeContext.java:480) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1267) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask$DelegatedAction.run(SSLEngineImpl.java:1254) ~[?:?]",
"at java.security.AccessController.doPrivileged(AccessController.java:691) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl$DelegatedTask.run(SSLEngineImpl.java:1199) ~[?:?]",
"at io.netty.handler.ssl.SslHandler.runAllDelegatedTasks(SslHandler.java:1542) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.runDelegatedTasks(SslHandler.java:1556) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1440) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1267) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1314) ~[netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:501) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:440) ~[netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"... 16 more"] }
kjdfkdsf
  • 211
  • 1
  • 3
  • 7

1 Answers1

1

We solved this and vary nasty. Below is why...

Here is actual problem with Azure KeyVault

  • Prepare the selfsigned PFX file (with full chain).
  • The PFX must be password protected (although Elasticsearch examples doesn't say about it clearly) for complete chain
  • Verify full chain by running below command keytool -list -keystone yourFullChainCert.pfx -storetype pkcs12 -v
  • Upload password protected PFX (with full chain) to Azure KeyVault using portal
  • Download the PFX file and run the keytool again. You will not be able to verify the chain

Solution, a nasty workaround (nasty because ( need build my own image)

Image Building

  • Encrypt the transport/ssl PFX private key with aes256 alg
  • Create a private Elasticsearch image
  • In the container image, place all PEM files in /usr/share/elasticsearch/config/certs directory
    • Your transport PFX root ca cert PEM, example rootCA.cert.pem
    • Your transport PFX optional issuing ca cert PEM, example issuingCA.cert.pem
    • Your transport PFX private key encrypted with aes256 PEM, example tls.aes.key.pem
    • Your transport PFX cert PEM, example tls.cert.pem
    • Your ssl PFX private key encrypted with aes256 PEM, ssl.aes.key.pem
    • Your ssl PFX cert PEM, example ssl.cert.pem
  • Place AES256 encryption key in Azure KeyVault
  • Sync the keys using CSI driver, here is the code snippet certysnc.yaml file
  • Place certysnc.yaml file in Elasticseach helm charts templates directory
apiVersion: secrets-store.csi.x-k8s.io/v1alpha1
kind: SecretProviderClass
metadata:
  name: {{ .Values.secretProviderClassName }}
  namespace: {{ .Release. Namespace }} 
spec:
  provider: azure
  secretObjects:
  - secretName: elasticsearch-tls-ssl-cert-key 
    type: Opaque
    data:
    - objectName: es-tls-ssl-cert-key
      key: xpack.security.transport.ssl.secure_key_passphrase
    - objectName: es-tls-ssl-cert-key
      key: xpack.security.http.ssl.secure_key_passphrase
  - secretName: elastic-credentials
    type: Opaque
    data:
    - objectName: elastic-username
      key: username
    - objectName: elastic-user-password
      key: password
  parameters:
    usePodIdentity: "true"
    userAssignedIdentityID: {{ .Values.userAssignedIdentityID }}
    keyvaultName: {{ .Values.keyvaultName }}
    cloudName: "AzurePublicCloud"
    objects:  |
      array:
        - |
          objectName: elastic-user-password
          objectType: secret
          objectVersion: ""
        - |
          objectName: elastic-username
          objectType: secret
          objectVersion: ""
        - |
          objectName: es-tls-ssl-cert-key
          objectType: secret
          objectVersion: ""
    tenantId: {{ .Values.tenantId }}
    resourceGroup: {{ .Values.resourceGroup }}
    subscriptionId: {{ .Values.subscriptionId }}
kjdfkdsf
  • 211
  • 1
  • 3
  • 7