Jochen,
the -icl switch was the culprit. I was able to validate that when I added the cert. Also when I go to the url https://localhost:9200/_searchguard/authinfo
It shows me
{"user":"User [name=kibanaserver, roles=[], requestedTenant=null]","user_name":"kibanaserver","user_requested_tenant":null,"remote_address":"192.168.1.211:59054","backend_roles":[],"custom_attribute_names":[],"sg_roles":["sg_kibana_server","sg_own_index"],"sg_tenants":{"kibanaserver":true},"principal":null,"peer_certificates":"0","sso_logout_url":null}
So I am assuming everything is setup right. However when I open kibana up it still shows me this
If I see the kibana logs I see this (on a full system reboot) and kibana service start
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["status","plugin:searchguard@6.3.1-14-beta-1","info"],"pid":17889,"state":"green","message":"Status changed from yellow to green - Search Guard plugin initialised.","prevState":"yellow","prevMsg":"Search Guard system routes registered."}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["plugins","debug"],"pid":17889,"plugin":{"author":"Chris Cowan<chris@elastic.co>","name":"metrics","version":"kibana"},"message":"Initializing plugin metrics@kibana"}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["status","plugin:metrics@6.3.1","info"],"pid":17889,"state":"green","message":"Status changed from uninitialized to green - Ready","prevState":"uninitialized","prevMsg":"uninitialized"}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["plugins","debug"],"pid":17889,"plugin":{"author":"Yuri Astrakhan<yuri@elastic.co>","name":"vega","version":"kibana"},"message":"Initializing plugin vega@kibana"}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["plugins","debug"],"pid":17889,"plugin":{"name":"x-pack","version":"6.3.1","private":true,"author":"Elastic","license":"Elastic-License","kibana":{"build":{"intermediateBuildDirectory":"build/plugin/kibana/x-pack"},"version":"6.3.1"},"dependencies":{"@elastic/eui":"v0.0.38-bugfix.1","@elastic/node-crypto":"0.1.2","@elastic/node-phantom-simple":"2.2.4","@elastic/numeral":"2.3.2","@kbn/datemath":"file:../packages/kbn-datemath","@kbn/ui-framework":"file:../packages/kbn-ui-framework","angular-paging":"2.2.1","angular-resource":"1.4.9","angular-sanitize":"1.4.9","angular-ui-ace":"0.2.3","angular-ui-bootstrap":"1.2.5","babel-core":"^6.26.0","babel-preset-es2015":"^6.24.1","babel-runtime":"^6.26.0","bluebird":"3.1.1","boom":"3.1.1","brace":"0.10.0","chrome-remote-interface":"0.24.2","classnames":"2.2.5","concat-stream":"1.5.1","d3":"3.5.6","d3-scale":"1.0.6","dedent":"^0.7.0","dragselect":"1.7.17","elasticsearch":"13.0.1","extract-zip":"1.5.0","font-awesome":"4.4.0","get-port":"2.1.0","getos":"^3.1.0","glob":"6.0.4","hapi-auth-cookie":"6.1.1","history":"4.7.2","humps":"2.0.1","icalendar":"0.7.1","isomorphic-fetch":"2.2.1","joi":"6.10.1","jquery":"^3.3.1","jstimezonedetect":"1.0.5","lodash":"3.10.1","lodash.mean":"^4.1.0","lodash.orderby":"4.6.0","mkdirp":"0.5.1","moment":"^2.20.1","moment-duration-format":"^1.3.0","moment-timezone":"^0.5.14","ngreact":"^0.5.1","object-hash":"1.2.0","path-match":"1.2.4","pdfmake":"0.1.33","pivotal-ui":"13.0.1","pluralize":"3.1.0","pngjs":"3.3.1","prop-types":"^15.6.0","puid":"1.0.5","react":"^16.2.0","react-clipboard.js":"^1.1.2","react-dom":"^16.2.0","react-markdown-renderer":"^1.4.0","react-portal":"^3.2.0","react-redux":"^5.0.5","react-router-breadcrumbs-hoc":"1.1.2","react-router-dom":"^4.2.2","react-select":"^1.2.1","react-sticky":"^6.0.1","react-syntax-highlighter":"^5.7.0","react-vis":"^1.8.1","redux":"3.7.2","redux-actions":"2.2.1","redux-thunk":"2.2.0","request":"^2.85.0","reselect":"3.0.1","rimraf":"^2.6.2","rison-node":"0.3.1","rxjs":"5.3.0","semver":"5.1.0","stream-to-observable":"0.2.0","styled-components":"2.3.2","tar-fs":"1.13.0","tinycolor2":"1.3.0","ui-select":"0.19.4","unbzip2-stream":"1.0.9","uuid":"3.0.1","venn.js":"0.2.9","webcola":"3.3.6","xregexp":"3.2.0"},"engines":{"yarn":"^1.6.0"},"build":{"git":{"count":"17276","sha":"cb83982","date":"Fri, 29 Jun 2018 11:45:07 -0700"},"date":"Fri Jun 29 2018 22:11:04 GMT+0000 (UTC)"}},"message":"Initializing plugin reporting@6.3.1"}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["reporting","debug","exportTypes"],"pid":17889,"message":"Found exportType at /usr/share/kibana/node_modules/x-pack/plugins/reporting/export_types/csv/server/index.js"}
{"type":"log","@timestamp":"2018-08-16T21:33:31Z","tags":["reporting","debug","exportTypes"],"pid":17889,"message":"Found exportType at /usr/share/kibana/node_modules/x-pack/plugins/reporting/export_types/printable_pdf/server/index.js"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","warning"],"pid":17889,"message":"Generating a random key for xpack.reporting.encryptionKey. To prevent pending reports from failing on restart, please set xpack.reporting.encryptionKey in kibana.yml"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","debug"],"pid":17889,"message":"Browser type: phantom"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["status","plugin:reporting@6.3.1","info"],"pid":17889,"state":"yellow","message":"Status changed from uninitialized to yellow - Waiting for Elasticsearch","prevState":"uninitialized","prevMsg":"uninitialized"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["error","elasticsearch","admin"],"pid":17889,"message":"Request error, retrying\nHEAD https://localhost:9200/ => connect ECONNREFUSED 127.0.0.1:9200"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","debug"],"pid":17889,"message":"Running on os \"linux\", distribution \"Centos\", release \"7.4.1708\""}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","debug"],"pid":17889,"message":"Browser installed at /var/lib/kibana/phantomjs-2.1.1-linux-x86_64/bin/phantomjs"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","worker","debug"],"pid":17889,"message":"CSV: Registering CSV worker"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","esqueue","worker","debug"],"pid":17889,"message":"jkx30j5t0dsxb01c6817m9xm - Created worker for job type csv"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","worker","debug"],"pid":17889,"message":"PDF: Registering PDF worker"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["reporting","esqueue","worker","debug"],"pid":17889,"message":"jkx30j5u0dsxb01c6818bu1d - Created worker for job type printable_pdf"}
{"type":"log","@timestamp":"2018-08-16T21:33:32Z","tags":["fatal"],"pid":17889,"message":"Port 5601 is already in use. Another instance of Kibana may be running!"}
···
On Tuesday, August 14, 2018 at 3:45:04 PM UTC-4, Jochen Kressin wrote:
I’m inclined to say that’s not possible 
The error you are seeing is because sgadmin cannot talk to you cluster, which contradicts the statement that it ran successfully once. If there is a connection problem it will always show up, regardless of what commands and switches you use.
Are you sure the two commands (the succeeding one and the failing one) are exactly the same? Especially - are the settings for the cluster name the same? In your posted call I do not see the -icl (ignore cluster name) or the -cn (cluster name) switch. Can you check that again?
On Monday, August 13, 2018 at 4:27:18 PM UTC+2, Adwait Joshi wrote:
Jochen,
I was able to get past most of it using your tls troubleshooting guide. I have the cluster up and running on elastic side. I ran SGAdmin and it ran successfully but when I try to run the retrieve command I get the following error
On Monday, July 23, 2018 at 4:18:18 PM UTC-4, Jochen Kressin wrote:
Why did the installation with the TLS tool fail? The TLS Tool creates a snippet for each node that you just need to copy and paste to elasticsearch.yml as-is. After copying the generated certificates to the config folder the cluster should start without problems.
This error:
Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
``
means that the certificate you use in your sgadmin call could not be validated against the root (and intermediate certs if you have any) certificate. So there is something wrong in your certificate chain.
I suggest going through the steps in the TLS troubleshooting guide:
https://docs.search-guard.com/latest/troubleshooting-tls
And especially check if your admin certificate is actually signed by the root CA you have configured in elasticsearch.yml (transport section).
On Sunday, July 22, 2018 at 2:15:45 PM UTC+2, Adwait Joshi wrote:
I am using Elastic 6.3.X and Kibana 6.3.X so I installed the search guard plugin for Elastic. The Kibana version is still showing in beta.
I tried to install the certificates fromt he “TLS generator Tool” however that failed miserably. I was then able to generate my own certificates using OpenSSL and atleast get past the error in log files. However SGAdmin is not initialized, and when I try to initialize it I get the following error. Can someone help me get past it?
Unable to check whether cluster is sane: None of the configured nodes are available: [{#transport#-1}{PjEbtCKjSkuqFwv7O3pMow}{localhost}{127.0.0.1:9300}]
08:14:15.386 [elasticsearch[client][transport_client_boss][T#1]] ERROR com.floragunn.searchguard.ssl.transport.SearchGuardSSLNettyTransport - SSL Problem General OpenSslEngine problem
javax.net.ssl.SSLHandshakeException: General OpenSslEngine problem
at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:648) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.internal.tcnative.SSL.readFromSSL(Native Method) ~[netty-tcnative-2.0.12.Final-linux-x86_64-fedora.jar:2.0.12.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.readPlaintextData(ReferenceCountedOpenSslEngine.java:482) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1020) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1127) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslEngine.unwrap(ReferenceCountedOpenSslEngine.java:1170) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:215) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1215) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1127) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1162) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.16.Final.jar:4.1.16.Final]
at java.lang.Thread.run(Unknown Source) [?:1.8.0_144]
Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
at sun.security.validator.PKIXValidator.doBuild(Unknown Source) ~[?:1.8.0_144]
at sun.security.validator.PKIXValidator.engineValidate(Unknown Source) ~[?:1.8.0_144]
at sun.security.validator.Validator.validate(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.validate(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(Unknown Source) ~[?:1.8.0_144]
at io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:221) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:644) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
… 26 more
Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
at sun.security.provider.certpath.SunCertPathBuilder.build(Unknown Source) ~[?:1.8.0_144]
at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(Unknown Source) ~[?:1.8.0_144]
at java.security.cert.CertPathBuilder.build(Unknown Source) ~[?:1.8.0_144]
at sun.security.validator.PKIXValidator.doBuild(Unknown Source) ~[?:1.8.0_144]
at sun.security.validator.PKIXValidator.engineValidate(Unknown Source) ~[?:1.8.0_144]
at sun.security.validator.Validator.validate(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.validate(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(Unknown Source) ~[?:1.8.0_144]
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(Unknown Source) ~[?:1.8.0_144]
at io.netty.handler.ssl.ReferenceCountedOpenSslClientContext$ExtendedTrustManagerVerifyCallback.verify(ReferenceCountedOpenSslClientContext.java:221) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.ReferenceCountedOpenSslContext$AbstractCertificateVerifier.verify(ReferenceCountedOpenSslContext.java:644) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
… 26 more
ERR: Cannot connect to Elasticsearch. Please refer to elasticsearch logfile for more information
Trace:
NoNodeAvailableException[None of the configured nodes are available: [{#transport#-1}{PjEbtCKjSkuqFwv7O3pMow}{localhost}{127.0.0.1:9300}]]
at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:347)
at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:245)
at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:60)
at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:378)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405)
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:394)
at com.floragunn.searchguard.tools.SearchGuardAdmin.main0(SearchGuardAdmin.java:451)
at com.floragunn.searchguard.tools.SearchGuardAdmin.main(SearchGuardAdmin.java:124)
