Elasticsearch/Kibana 7.6.2 - Receiving error: "javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown"

Search Guard
1

TL;DR

I am receiving the following error in my Elasticsearch log when Kibana attempts to connect with TLS enabled:

javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown

Full Story

I’m receiving this error in my Elasticsearch log when Kibana attempts to connect with TLS enabled. As far as I can tell, everything is configured correctly. My config files and error log follow. All certificates are issued and signed by our company, and elasticsearch.ssl.certificateAuthorities contains the full certificate chain.

Edit: This is being deployed via Kubernetes, not on servers with static IP addresses.

elasticsearch.yml

action:
  auto_create_index: ".*,sg7-*,searchguard,searchguard*"

# Allow both primary and non-primary shards to be moved between nodes for
# balancing purposes.
cluster:
  routing.allocation.enable: all

xpack:
  # Disable xpack.security when using SearchGuard.
  security.enabled: false
  # Cluster monitoring configuration.
  monitoring:
    enabled: true
    collection:
      enabled: true

# SearchGuard configuration.
searchguard:
  # Next line is for use in sandbox only.
  # allow_unsafe_democertificates: true

  # nodes_dn:
  #   - ".*"
  allow_default_init_sgindex: true
  audit:
    type: internal_elasticsearch

  authcz:
    admin_dn:
      - EMAILADDRESS=pki@example.com,CN=elk.example.com,OU=Example USA,O=Example USA,L=Smallville,ST=Kansas,C=US

  check_snapshot_restore_write_privileges: true
  enable_snapshot_restore_privilege: true

  restapi:
    roles_enabled:
      - SGS_ALL_ACCESS
  
  # Searchguard config
  ssl:
    http:
      # Mount certs to /usr/share/elasticsearch/config/certs
      enabled: true
      pemcert_filepath: certs/cert.pem
      pemkey_filepath: certs/key-p8.pem
      pemtrustedcas_filepath: certs/ca_bundle.pem
      clientauth_mode: OPTIONAL

      enabled_protocols:
        - TLSv1.3
        - TLSv1.2
        - TLSv1.1
    
    transport:
      # Mount certs to /usr/share/elasticsearch/config
      pemcert_filepath: certs/cert.pem
      pemkey_filepath: certs/key-p8.pem
      pemtrustedcas_filepath: certs/ca_bundle.pem
      enforce_hostname_verification: false

kibana.yml

# Xpack configuration.
xpack:
  # Configure xpack security.
  security.enabled: false

elasticsearch:
  username: "${SG_KIBANASERVER_USER}"
  password: "${SG_KIBANASERVER_PASSWD}"
  ssl:
    alwaysPresentCertificate: false
    certificate: /usr/share/kibana/config/certs/cert.pem
    certificateAuthorities: 
      # - /usr/share/kibana/config/certs/ca_bundle.pem
      - /usr/share/kibana/config/certs/ca_root.pem
      - /usr/share/kibana/config/certs/ca_int.pem
    key: /usr/share/kibana/config/certs/key.pem
    verificationMode: none

Error Log

{"type": "server", "timestamp": "2020-04-21T21:15:59,059Z", "level": "WARN", "component": "o.e.h.AbstractHttpServerTransport", "cluster.name": "es", "node.name": "es-master-0", "message": "caught exception while handling client http traffic, closing connection Netty4HttpChannel{localAddress=/10.229.94.166:9200, remoteAddress=/10.229.91.23:48970}", "cluster.uuid": "hvkuxjblQ3WsHjz8ON35-g", "node.id": "_tl82_dSQBiOxnZBzJaGXQ" , 
"stacktrace": ["io.netty.handler.codec.DecoderException: javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:473) ~[netty-codec-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:281) ~[netty-codec-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:700) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:600) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:554) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:514) [netty-transport-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.43.Final.jar:4.1.43.Final]",
"at java.lang.Thread.run(Thread.java:830) [?:?]",
"Caused by: javax.net.ssl.SSLHandshakeException: Received fatal alert: certificate_unknown",
"at sun.security.ssl.Alert.createSSLException(Alert.java:131) ~[?:?]",
"at sun.security.ssl.Alert.createSSLException(Alert.java:117) ~[?:?]",
"at sun.security.ssl.TransportContext.fatal(TransportContext.java:311) ~[?:?]",
"at sun.security.ssl.Alert$AlertConsumer.consume(Alert.java:291) ~[?:?]",
"at sun.security.ssl.TransportContext.dispatch(TransportContext.java:184) ~[?:?]",
"at sun.security.ssl.SSLTransport.decode(SSLTransport.java:164) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl.decode(SSLEngineImpl.java:729) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:684) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:499) ~[?:?]",
"at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:475) ~[?:?]",
"at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:634) ~[?:?]",
"at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:280) ~[netty-handler-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1332) ~[netty-handler-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1227) ~[netty-handler-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1274) ~[netty-handler-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:503) ~[netty-codec-4.1.43.Final.jar:4.1.43.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:442) ~[netty-codec-4.1.43.Final.jar:4.1.43.Final]",
"... 16 more"] }
2

Check the IP address in the certificates. Maybe the IP address in your certificate is not the address which is communicating with the Elasticsearch. It can happen if a host has multiple network interfaces or have IPv4 and IPv6 on the same network interface. Let me know.

Also, you can find more about TLS troubleshooting in this article TLS help | Security for Elasticsearch | Search Guard

3

I’ll certainly take a look at that. The other item that I probably should’ve included in the initial post (which I’ve now edited it to include) is that I’m deploying this via Kubernetes. Any best practices or specific recommendations there?

4

The issue is resolved. It appears that the majority of the errors were because I had Logstash running, but hadn’t yet configured it to use TLS. Once I stopped Logstash, the majority of these issues stopped.

I then updated my kibana.yml to the following:

server:
  xsrf:
    whitelist:
      - "/searchguard/saml/acs/idpinitiated"
      - "/searchguard/saml/acs"
      - "/searchguard/saml/logout"

# Xpack configuration.
xpack:
  # Configure xpack security.
  security.enabled: false

elasticsearch:
  username: "${SG_KIBANASERVER_USER}"
  password: "${SG_KIBANASERVER_PASSWD}"
  ssl:
    alwaysPresentCertificate: false
    certificate: /usr/share/kibana/config/certs/node/cert.pem
    key: /usr/share/kibana/config/certs/node/key.pem
    certificateAuthorities: 
      - /usr/share/kibana/config/certs/node/ca_bundle.pem
    verificationMode: none

At that point, I stopped receiving all errors from Elasticsearch, and it appeared that Kibana was successfully communicating with it. Please note that I am still receiving a 302 error from the Kibana readiness probe and cannot connect to it via a web browser, but I am addressing that issue in a separate post.

When I run the following command to retrieve the most recent document in the .monitoring-kibana-7-* indices…

curl -k -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD -H 'Content-Type: application/json' \
  https://localhost:9200/.monitoring-kibana-7-*/_search?pretty -d '
    {
      "query": {
        "match_all": {}
      },
      "size": 1,
      "sort": [
        {
          "timestamp": {
            "order": "desc"
          }
        }
      ]
    }'

…I receive the following output, which appears to indicate that Kibana is communicating to Elasticsearch on an ongoing basis (see timestamp):

{
  "took" : 58,
  "timed_out" : false,
  "_shards" : {
    "total" : 7,
    "successful" : 7,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [
      {
        "_index" : ".monitoring-kibana-7-2020.04.23",
        "_type" : "_doc",
        "_id" : "tIvwp3EBjwb6QOtkyCo0",
        "_score" : null,
        "_source" : {
          "cluster_uuid" : "hvkuxjblQ3WsHjz8ON35-g",
          "timestamp" : "2020-04-23T16:48:27.185Z",
          "interval_ms" : 10000,
          "type" : "kibana_stats",
          "source_node" : {
            "uuid" : "urvAbq76SqCjnAEPsr_hyA",
            "host" : "10.229.94.58",
            "transport_address" : "10.229.94.58:9300",
            "ip" : "10.229.94.58",
            "name" : "elk-es-coord-0",
            "timestamp" : "2020-04-23T16:48:27.185Z"
          },
          "kibana_stats" : {
            "kibana" : {
              "uuid" : "5ddddc36-822d-4c5e-aa2d-2aa1c883f025",
              "name" : "elk-kibana-6b7d4d7dc6-cn9xj",
              "index" : ".kibana",
              "host" : "0.0.0.0",
              "transport_address" : "0.0.0.0:5601",
              "version" : "7.6.2",
              "snapshot" : false,
              "status" : "red"
            },
            "cloud" : {
              "name" : "aws",
              "id" : "i-000c8043efc0be372",
              "vm_type" : "r5d.4xlarge",
              "region" : "us-east-1",
              "zone" : "us-east-1c",
              "metadata" : {
                "architecture" : "x86_64",
                "marketplaceProductCodes" : null,
                "imageId" : "ami-0dc7713312a7ec987",
                "kernelId" : null,
                "pendingTime" : "2020-04-02T18:17:30Z",
                "ramdiskId" : null,
                "version" : "2017-09-30"
              }
            },
            "concurrent_connections" : 0,
            "os" : {
              "load" : {
                "1m" : 3.01611328125,
                "5m" : 2.9482421875,
                "15m" : 2.87060546875
              },
              "memory" : {
                "total_in_bytes" : 133653843968,
                "free_in_bytes" : 120397815808,
                "used_in_bytes" : 13256028160
              },
              "uptime_in_millis" : 747809000,
              "platform" : "linux",
              "platformRelease" : "linux-4.14.171-136.231.amzn2.x86_64",
              "distro" : "Centos",
              "distroRelease" : "Centos-7.7.1908"
            },
            "process" : {
              "event_loop_delay" : 1.523249626159668,
              "memory" : {
                "heap" : {
                  "total_in_bytes" : 858374144,
                  "used_in_bytes" : 599807576,
                  "size_limit" : 2217857988
                },
                "resident_set_size_in_bytes" : 1243553792
              },
              "uptime_in_millis" : 9496096
            },
            "requests" : {
              "disconnects" : 0,
              "total" : 2
            },
            "response_times" : {
              "average" : 2,
              "max" : 2
            },
            "timestamp" : "2020-04-23T16:48:24.960Z"
          }
        },
        "sort" : [
          1587660507185
        ]
      }
    ]
  }
}
1 Like
5

This topic was automatically closed 21 days after the last reply. New replies are no longer allowed.