I have followed below steps to start elasticsearch 7.3.0 with search guard version-36.1.0 as mentioned in Latest Releases | Security for Elasticsearch | Search Guard
- created elasticsearch image from elasticsearch-7.x.repo yum repo with user as “elastic”.
- installed search guard plugin from https://oss.sonatype.org/service/local/repositories/releases/content/com/floragunn/search-guard-7/7.3.0-36.1.0/search-guard-7-7.3.0-36.1.0.zip
- added openssl dependency for Search Guard
openssl netty-tcnative-openssl-1.1.0j-static-2.0.20.Final-non-fedora-linux-x86_64.jar
When elasticsearch is trying to start
getting below error:
Caused by: java.lang.NoClassDefFoundError: io/netty/internal/tcnative/SSLPrivateKeyMethod
Full log:
[2019-09-13T05:23:37,148][WARN ][org.elasticsearch.bootstrap.ElasticsearchUncaughtExceptionHandler] uncaught exception in thread [main]
org.elasticsearch.bootstrap.StartupException: java.lang.IllegalStateException: failed to load plugin class [com.floragunn.searchguard.SearchGuardPlugin]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:163) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.execute(Elasticsearch.java:150) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.cli.EnvironmentAwareCommand.execute(EnvironmentAwareCommand.java:86) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.cli.Command.mainWithoutErrorHandling(Command.java:124) ~[elasticsearch-cli-7.3.0.jar:7.3.0]
at org.elasticsearch.cli.Command.main(Command.java:90) ~[elasticsearch-cli-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:115) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:92) ~[elasticsearch-7.3.0.jar:7.3.0]
Caused by: java.lang.IllegalStateException: failed to load plugin class [com.floragunn.searchguard.SearchGuardPlugin]
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:614) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundle(PluginsService.java:556) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundles(PluginsService.java:471) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.(PluginsService.java:163) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:314) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:258) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap$5.(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:349) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:159) ~[elasticsearch-7.3.0.jar:7.3.0]
… 6 more
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_201]
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:605) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundle(PluginsService.java:556) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundles(PluginsService.java:471) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.(PluginsService.java:163) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:314) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:258) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap$5.(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:349) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:159) ~[elasticsearch-7.3.0.jar:7.3.0]
… 6 more
Caused by: java.lang.NoClassDefFoundError: io/netty/internal/tcnative/SSLPrivateKeyMethod
at io.netty.handler.ssl.SslContext.newServerContextInternal(SslContext.java:423) ~[?:?]
at io.netty.handler.ssl.SslContextBuilder.build(SslContextBuilder.java:447) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore$1.run(DefaultSearchGuardKeyStore.java:785) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore$1.run(DefaultSearchGuardKeyStore.java:782) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_201]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.buildSSLContext0(DefaultSearchGuardKeyStore.java:782) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.buildSSLServerContext(DefaultSearchGuardKeyStore.java:730) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.initSSLConfig(DefaultSearchGuardKeyStore.java:315) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.(DefaultSearchGuardKeyStore.java:151) ~[?:?]
at com.floragunn.searchguard.ssl.SearchGuardSSLPlugin.(SearchGuardSSLPlugin.java:194) ~[?:?]
at com.floragunn.searchguard.SearchGuardPlugin.(SearchGuardPlugin.java:212) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_201]
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:605) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundle(PluginsService.java:556) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundles(PluginsService.java:471) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.(PluginsService.java:163) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:314) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:258) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap$5.(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:349) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:159) ~[elasticsearch-7.3.0.jar:7.3.0]
… 6 more
Caused by: java.lang.ClassNotFoundException: io.netty.internal.tcnative.SSLPrivateKeyMethod
at java.net.URLClassLoader.findClass(URLClassLoader.java:382) ~[?:1.8.0_201]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[?:1.8.0_201]
at java.net.FactoryURLClassLoader.loadClass(URLClassLoader.java:817) ~[?:1.8.0_201]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[?:1.8.0_201]
at io.netty.handler.ssl.SslContext.newServerContextInternal(SslContext.java:423) ~[?:?]
at io.netty.handler.ssl.SslContextBuilder.build(SslContextBuilder.java:447) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore$1.run(DefaultSearchGuardKeyStore.java:785) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore$1.run(DefaultSearchGuardKeyStore.java:782) ~[?:?]
at java.security.AccessController.doPrivileged(Native Method) ~[?:1.8.0_201]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.buildSSLContext0(DefaultSearchGuardKeyStore.java:782) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.buildSSLServerContext(DefaultSearchGuardKeyStore.java:730) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.initSSLConfig(DefaultSearchGuardKeyStore.java:315) ~[?:?]
at com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore.(DefaultSearchGuardKeyStore.java:151) ~[?:?]
at com.floragunn.searchguard.ssl.SearchGuardSSLPlugin.(SearchGuardSSLPlugin.java:194) ~[?:?]
at com.floragunn.searchguard.SearchGuardPlugin.(SearchGuardPlugin.java:212) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?]
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_201]
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:605) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundle(PluginsService.java:556) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.loadBundles(PluginsService.java:471) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.plugins.PluginsService.(PluginsService.java:163) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:314) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.node.Node.(Node.java:258) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap$5.(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.setup(Bootstrap.java:221) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:349) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.bootstrap.Elasticsearch.init(Elasticsearch.java:159) ~[elasticsearch-7.3.0.jar:7.3.0]
… 6 more
After verifying the binaries
added netty-tcnative-2.0.25.Final.jar and removed the openssl netty-tcnative-openssl-1.1.0j-static-2.0.20.Final-non-fedora-linux-x86_64.jar
then elasticsearch started with message
[com.floragunn.searchguard.ssl.DefaultSearchGuardKeyStore] OpenSSL not available (this is not an error, we simply fallback to built-in JDK SSL) because of java.lang.IllegalArgumentException: Failed to load any of the given libraries: [netty_tcnative_linux_x86_64, netty_tcnative_linux_x86_64_fedora, netty_tcnative_x86_64, netty_tcnative].
Application pods are unable to communicate with elasticsearch via searchguard transport client.
error log from ES data nodeelastic-client-0.log (55.5 KB) elastic-data-0.log (2.5 MB) :
[org.elasticsearch.transport.TcpTransport] exception caught on transport layer [Netty4TcpChannel{localAddress=/192.168.0.243:9301, remoteAddress=/192.168.0.254:53524}], closing connection
java.lang.IllegalStateException: Received handshake message from unsupported version: [5.0.0] minimal compatible version is: [6.8.0]
at org.elasticsearch.transport.InboundMessage.ensureVersionCompatibility(InboundMessage.java:137) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.InboundMessage.access$000(InboundMessage.java:39) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.InboundMessage$Reader.deserialize(InboundMessage.java:76) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:116) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:105) ~[elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:660) [elasticsearch-7.3.0.jar:7.3.0]
at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) [transport-netty4-client-7.3.0.jar:7.3.0]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374) [netty-transport-4.1.36.Final.jar:4.1.36.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360) [netty-transport-4.1.36.Final.jar:4.1.36.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352) [netty-transport-4.1.36.Final.jar:4.1.36.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:323) [netty-codec-4.1.36.Final.jar:4.1.36.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:297) [netty-codec-4.1.36.Final.jar:4.1.36.Final]