When Search Guard SSL plugin is used in ElasticSearch 2.3.3 transport client. We are getting below exception
in the application
Stack Trace
2019-08-14 07:22:00, 127 xyz@abc.com WARN GenericExceptionMapper:38 - [thread=http-apr-8080-exec-5] [requestId=1b5fc5d4-1a0b-4804-8e9a-8153f9c77969]- Error while executing service
ElasticsearchException[Failed to load plugin class [com.floragunn.searchguard.ssl.SearchGuardSSLPlugin]]; nested: InvocationTargetException; nested: NoSuchMethodError[org.apache.tomcat.jni.SSL.newSSL(JZ)J];
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:483)
at org.elasticsearch.plugins.PluginsService.(PluginsService.java:102)
at org.elasticsearch.client.transport.TransportClient$Builder.build(TransportClient.java:126)
at com.myorg.elasticsearch.connection.ElasticSearchTransportClientFactory.init(ElasticSearchTransportClientFactory.java:63)
at com.myorg.elasticsearch.connection.ElasticSearchTransportClientFactory.getElasticClient(ElasticSearchTransportClientFactory.java:78)
at com.myorg.elasticsearch.indices.ElasticSearchIndexSearchWorker.getDocumentIdsCount(ElasticSearchIndexSearchWorker.java:682)
at com.kumbay.api.impl.ESQueryImpl.getRecordCountFor(ESQueryImpl.java:591)
at com.kumbay.service.ESQueryService.getRecordCountFor(ESQueryService.java:456)
at com.kumbay.service.ESQueryService$$FastClassBySpringCGLIB$$e8259dbb.invoke()
at org.springframework.cglib.proxy.MethodProxy.invoke(MethodProxy.java:204)
at org.springframework.aop.framework.CglibAopProxy$CglibMethodInvocation.invokeJoinpoint(CglibAopProxy.java:738)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:157)
at org.springframework.security.access.intercept.aopalliance.MethodSecurityInterceptor.invoke(MethodSecurityInterceptor.java:69)
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179)
at org.springframework.aop.framework.CglibAopProxy$DynamicAdvisedInterceptor.intercept(CglibAopProxy.java:673)
at com.kumbay.service.ESQueryService$$EnhancerBySpringCGLIB$$579b0e92.getRecordCountFor()
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:189)
at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:99)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:59)
at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:96)
at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:308)
at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:254)
at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:234)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:208)
at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:160)
at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:180)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:299)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doGet(AbstractHTTPServlet.java:223)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:624)
at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:274)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.HttpHeaderSecurityFilter.doFilter(HttpHeaderSecurityFilter.java:126)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:317)
at com.kumbay.filter.myorgAuthenticationProcessingFilter.successfulAuthentication(myorgAuthenticationProcessingFilter.java:442)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:240)
at com.kumbay.filter.myorgAuthenticationProcessingFilter.doFilter(myorgAuthenticationProcessingFilter.java:181)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:127)
at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:91)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:114)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:137)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:111)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.rememberme.RememberMeAuthenticationFilter.doFilter(RememberMeAuthenticationFilter.java:150)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:170)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:63)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilterInternal(BasicAuthenticationFilter.java:158)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.AbstractAuthenticationProcessingFilter.doFilter(AbstractAuthenticationProcessingFilter.java:200)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.authentication.logout.LogoutFilter.doFilter(LogoutFilter.java:116)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.header.HeaderWriterFilter.doFilterInternal(HeaderWriterFilter.java:66)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.context.request.async.WebAsyncManagerIntegrationFilter.doFilterInternal(WebAsyncManagerIntegrationFilter.java:56)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:105)
at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:331)
at org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:214)
at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:177)
at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:347)
at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:263)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.filters.ExpiresFilter.doFilter(ExpiresFilter.java:1203)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1136)
at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
at org.apache.tomcat.util.net.AprEndpoint$SocketWithOptionsProcessor.run(AprEndpoint.java:2492)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.reflect.InvocationTargetException
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.elasticsearch.plugins.PluginsService.loadPlugin(PluginsService.java:472)
… 103 more
Caused by: java.lang.NoSuchMethodError: org.apache.tomcat.jni.SSL.newSSL(JZ)J
at io.esshaded.netty.handler.ssl.OpenSsl.(OpenSsl.java:103)
at com.floragunn.searchguard.ssl.SearchGuardSSLPlugin$1.run(SearchGuardSSLPlugin.java:70)
at java.security.AccessController.doPrivileged(Native Method)
at com.floragunn.searchguard.ssl.SearchGuardSSLPlugin.(SearchGuardSSLPlugin.java:66)
ES version: 2.3.3
Search Guard SSL plugin: 2.3.3.12
We are using docker image for deploying the application
Dockerfile
FROM tomcat:7-jre8
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/*
&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
ENV LANG en_US.utf8
RUN apt-get update && apt-get install -y lsb-release apt-utils kmod
RUN echo “deb Index of /apt// gcsfuse-$(lsb_release -c -s) main” | tee /etc/apt/sources.list.d/gcsfuse.list
RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - ;
apt-get update && apt-get install -y gcsfuse
ENV GCLOUD_FILE_STORAGE_MOUNT “/media/gcloudstorage”
RUN mkdir -p $GCLOUD_FILE_STORAGE_MOUNT
RUN mkdir -p /opt/maven/lib
ADD lib/org.jacoco.agent-0.7.7.201606060606-runtime.jar /opt/maven/lib/org.jacoco.agent-0.7.7.201606060606-runtime.jar
ADD conf/spark.allocation.xml $CATALINA_HOME/
RUN mkdir $CATALINA_HOME/dataexportcsv
ADD conf/key.json $CATALINA_HOME/dataexportcsv/
ADD conf/tempkey.json $CATALINA_HOME/dataexportcsv/
ADD conf/server.xml $CATALINA_HOME/conf
ADD conf/web.xml $CATALINA_HOME/conf
ADD binaries/ws.war $CATALINA_HOME/webapps/server.war
ADD binaries/vishesh-cassandra-1.0-worker.jar $CATALINA_HOME/
ENV CATALINA_OPTS=“-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7200 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false”
ENTRYPOINT [“catalina.sh”, “run”]
CMD [“”]
Openssl version 1.1.0f
Host OS Details:
Distributor ID: Debian
Description: Debian GNU/Linux 9.5 (stretch)
Release: 9.5
Codename: stretch
Search Guard SSL config in Elasticsearch.yml
searchguard.authcz.admin_dn:
CN=******,OU=client,O=client,L=test, C=de
CN=*******,OU=client,O=client,L=test, C=de
##--------------HTTP layer SSL--------------##
searchguard.ssl.http.enabled: true
searchguard.ssl.http.keystore_type: PKCS12
searchguard.ssl.http.keystore_password: *******
searchguard.ssl.http.keystore_filepath: node-0-keystore.p12
searchguard.ssl.http.truststore_type: PKCS12
searchguard.ssl.http.truststore_password: ********
searchguard.ssl.http.truststore_filepath: truststore.p12
searchguard.ssl.http.enforce_hostname_verification: false
searchguard.ssl.http.enable_openssl_if_available: true
##----------------transport SSL--------------------##
searchguard.ssl.transport.keystore_type: PKCS12
searchguard.ssl.transport.keystore_password: ********
searchguard.ssl.transport.keystore_filepath: node-0-keystore.p12
searchguard.ssl.transport.truststore_type: PKCS12
searchguard.ssl.transport.truststore_password: *********
searchguard.ssl.transport.truststore_filepath: truststore.p12
searchguard.ssl.transport.enforce_hostname_verification: false
searchguard.ssl.transport.enable_openssl_if_available: true
#searchguard.ssl.transport.enforce_hostname_verification: true
#searchguard.ssl.transport.resolve_hostname:true
searchguard.restapi.roles_enabled: [“SGS_ALL_ACCESS”]