Elasticsearch 7.7.1
Searchguard 42
We are running into the below error when using Moloch. It seems that this is a known issue and that there was a bug issue opened with SearchGuard to resolve the issue? Checking to see the status.
[2020-06-17T16:49:43,213][DEBUG][o.e.a.s.TransportSearchAction] [server-es-01] [sessions2-200616h00][0], node[yIBv2q9GSQiNexzW33UBSw], [P], s[STARTED], a[id=VHCUbdRZT6e497M272ApBg]: Failed to execute [SearchRequest{searchType=QUERY_THEN_FETCH, indices=[sessions2-200616h12, sessions2-200617h12, sessions2-200617h00, sessions2-200616h00, sessions2-200616h18, sessions2-200617h18, sessions2-200617h06, sessions2-200616h06], indicesOptions=IndicesOptions[ignore_unavailable=true, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true], types=[], routing='null', preference='primaries', requestCache=null, scroll=null, maxConcurrentShardRequests=0, batchedReduceSize=512, preFilterShardSize=null, allowPartialSearchResults=true, localClusterAlias=null, getOrCreateAbsoluteStartMillis=-1, ccsMinimizeRoundtrips=true, source={"size":50,"timeout":"300s","query":{"bool":{"filter":[{"range":{"lastPacket":{"from":1592283600000,"to":1592430560000,"include_lower":true,"include_upper":true,"boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":["ipProtocol","firstPacket","lastPacket","srcIp","srcGEO","srcPort","dstIp","dstGEO","dstPort","totPackets","totDataBytes","totBytes","node","http.uri","email.src","email.dst","email.subject","email.filename","dns.host","cert.alt","irc.channel"],"excludes":[]},"sort":[{"firstPacket":{"order":"desc","missing":"_first","unmapped_type":"string"}}],"track_total_hits":2147483647,"aggregations":{"dbHisto":{"histogram":{"field":"lastPacket","interval":60000.0,"offset":0.0,"order":{"_key":"asc"},"keyed":false,"min_doc_count":1},"aggregations":{"srcPackets":{"sum":{"field":"srcPackets"}},"dstPackets":{"sum":{"field":"dstPackets"}},"srcBytes":{"sum":{"field":"srcBytes"}},"dstBytes":{"sum":{"field":"dstBytes"}},"srcDataBytes":{"sum":{"field":"srcDataBytes"}},"dstDataBytes":{"sum":{"field":"dstDataBytes"}}}}}}}] lastShard [true]
org.elasticsearch.transport.TransportException: failure to send
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:628) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:672) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.SearchTransportService.sendCanMatch(SearchTransportService.java:114) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.CanMatchPreFilterSearchPhase.executePhaseOnShard(CanMatchPreFilterSearchPhase.java:82) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:231) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:266) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:400) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.access$100(AbstractSearchAsyncAction.java:68) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction$1.onFailure(AbstractSearchAsyncAction.java:245) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.ActionListenerResponseHandler.handleException(ActionListenerResponseHandler.java:59) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:630) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.transport.TransportService.sendChildRequest(TransportService.java:672) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.SearchTransportService.sendCanMatch(SearchTransportService.java:114) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.CanMatchPreFilterSearchPhase.executePhaseOnShard(CanMatchPreFilterSearchPhase.java:82) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.lambda$performPhaseOnShard$3(AbstractSearchAsyncAction.java:231) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:266) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:202) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:350) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:169) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:518) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:403) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.TransportSearchAction.lambda$doExecute$3(TransportSearchAction.java:213) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:114) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:87) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:244) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:88) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:153) [elasticsearch-7.7.1.jar:7.7.1]
at com.floragunn.searchguard.filter.SearchGuardFilter.apply0(SearchGuardFilter.java:279) [sg-suite-security-7.7.1-42.0.0.jar:7.7.1-42.0.0]
at com.floragunn.searchguard.filter.SearchGuardFilter.apply(SearchGuardFilter.java:114) [sg-suite-security-7.7.1-42.0.0.jar:7.7.1-42.0.0]
at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:151) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:129) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:64) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:90) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:399) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:117) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:115) [elasticsearch-7.7.1.jar:7.7.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:81) [sg-suite-security-7.7.1-42.0.0.jar:7.7.1-42.0.0]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:236) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:318) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:176) [elasticsearch-7.7.1.jar:7.7.1]
at com.floragunn.searchguard.ssl.http.netty.ValidatingDispatcher.dispatchRequest(ValidatingDispatcher.java:63) [sg-suite-security-7.7.1-42.0.0.jar:7.7.1-42.0.0]
at org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:329) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:383) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:308) [elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:54) [transport-netty4-client-7.7.1.jar:7.7.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:29) [transport-netty4-client-7.7.1.jar:7.7.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:58) [transport-netty4-client-7.7.1.jar:7.7.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:89) [transport-netty4-client-7.7.1.jar:7.7.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1470) [netty-handler-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1219) [netty-handler-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1266) [netty-handler-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:498) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:437) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) [netty-codec-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.45.Final.jar:4.1.45.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.45.Final.jar:4.1.45.Final]
at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.45.Final.jar:4.1.45.Final]
at java.lang.Thread.run(Thread.java:832) [?:?]
Caused by: java.lang.IllegalArgumentException: value for key [X-Opaque-Id] already present
at org.elasticsearch.common.util.concurrent.ThreadContext$ThreadContextStruct.putSingleHeader(ThreadContext.java:495) ~[elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ThreadContextStruct.putHeaders(ThreadContext.java:505) ~[elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.common.util.concurrent.ThreadContext$ThreadContextStruct.access$200(ThreadContext.java:440) ~[elasticsearch-7.7.1.jar:7.7.1]
at org.elasticsearch.common.util.concurrent.ThreadContext.putHeader(ThreadContext.java:328) ~[elasticsearch-7.7.1.jar:7.7.1]
at com.floragunn.searchguard.transport.SearchGuardInterceptor.sendRequestDecorate(SearchGuardInterceptor.java:166) ~[?:?]
at com.floragunn.searchguard.SearchGuardPlugin$7$1.sendRequest(SearchGuardPlugin.java:683) ~[?:?]
at org.elasticsearch.transport.TransportService.sendRequest(TransportService.java:621) ~[elasticsearch-7.7.1.jar:7.7.1]
... 102 more