Hi,
I am using ES 6.5.4 with three node(10.107.122.81-83)
SearchGuard 6.5.4-24.3.
when I install the SearchGuard plugin on node2(10.107.122.82). the ES fails to start with below error messages.
[2019-04-25T08:26:43,242][WARN ][o.e.c.l.LogConfigurator ] [sgt-002] Some logging configurations have %marker but don’t have %node_name. We will automatically add %node_name to the pattern to ease the migration for users who customize log4j2.properties but will stop this behavior in 7.0. You should manually replace %node_name with [%node_name]%marker in these locations:
/etc/elasticsearch/log4j2.properties
[2019-04-25T08:26:43,501][INFO ][o.e.e.NodeEnvironment ] [sgt-002] using [1] data paths, mounts [[/ (rootfs)]], net usable_space [43.1gb], net total_space [49.9gb], types [rootfs]
[2019-04-25T08:26:43,501][INFO ][o.e.e.NodeEnvironment ] [sgt-002] heap size [3.9gb], compressed ordinary object pointers [true]
[2019-04-25T08:26:43,534][INFO ][o.e.n.Node ] [sgt-002] node name [sgt-002], node ID [GnDUbCVARLGpnnsjUifJPA]
[2019-04-25T08:26:43,534][INFO ][o.e.n.Node ] [sgt-002] version[6.5.4], pid[3975], build[default/tar/d2ef93d/2018-12-17T21:17:40.758843Z], OS[Linux/3.10.0-957.el7.x86_64/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_201/25.201-b09]
[2019-04-25T08:26:43,535][INFO ][o.e.n.Node ] [sgt-002] JVM arguments [-Xms4096m, -Xmx4096m, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -XX:+HeapDumpOnOutOfMemoryError, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/etc/elasticsearch, -Des.distribution.flavor=default, -Des.distribution.type=tar]
[2019-04-25T08:26:45,316][INFO ][c.f.s.SearchGuardPlugin ] [sgt-002] ES Config path is /etc/elasticsearch
[2019-04-25T08:26:45,317][DEBUG][c.f.s.SearchGuardPlugin ] [sgt-002] Client side initiated TLS renegotiation forcibly disabled. This can prevent DoS attacks. (jdk.tls.rejectClientInitiatedRenegotiation set to true).
[2019-04-25T08:26:45,363][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] OpenSSL not available (this is not an error, we simply fallback to built-in JDK SSL) because of java.lang.ClassNotFoundException: io.netty.internal.tcnative.SSL
[2019-04-25T08:26:45,369][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] JVM supports the following 4 protocols [SSLv2Hello, TLSv1, TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,369][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] JVM supports the following 42 ciphers [TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_RSA_WITH_AES_256_CBC_SHA256, TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384, TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_RSA_WITH_AES_256_CBC_SHA, TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA, TLS_ECDH_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_RSA_WITH_AES_128_CBC_SHA256, TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256, TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, TLS_RSA_WITH_AES_128_CBC_SHA, TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDH_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_DSS_WITH_AES_128_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_128_GCM_SHA256]
[2019-04-25T08:26:45,370][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Config directory is /etc/elasticsearch/, from there the key- and truststore files are resolved relatively
[2019-04-25T08:26:45,370][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.transport.pemcert_filepath is esnode.pem
[2019-04-25T08:26:45,370][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved esnode.pem to /etc/elasticsearch/esnode.pem against /etc/elasticsearch
[2019-04-25T08:26:45,370][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.transport.pemkey_filepath is esnode-key.pem
[2019-04-25T08:26:45,370][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved esnode-key.pem to /etc/elasticsearch/esnode-key.pem against /etc/elasticsearch
[2019-04-25T08:26:45,371][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.transport.pemtrustedcas_filepath is root-ca.pem
[2019-04-25T08:26:45,371][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved root-ca.pem to /etc/elasticsearch/root-ca.pem against /etc/elasticsearch
[2019-04-25T08:26:45,438][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.http.pemtrustedcas_filepath is root-ca.pem
[2019-04-25T08:26:45,439][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved root-ca.pem to /etc/elasticsearch/root-ca.pem against /etc/elasticsearch
[2019-04-25T08:26:45,439][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.http.pemcert_filepath is esnode.pem
[2019-04-25T08:26:45,439][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved esnode.pem to /etc/elasticsearch/esnode.pem against /etc/elasticsearch
[2019-04-25T08:26:45,439][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Value for searchguard.ssl.http.pemkey_filepath is esnode-key.pem
[2019-04-25T08:26:45,439][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Resolved esnode-key.pem to /etc/elasticsearch/esnode-key.pem against /etc/elasticsearch
[2019-04-25T08:26:45,442][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] TLS Transport Client Provider : JDK
[2019-04-25T08:26:45,442][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] TLS Transport Server Provider : JDK
[2019-04-25T08:26:45,442][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] TLS HTTP Provider : JDK
[2019-04-25T08:26:45,442][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslTransportClientProvider:JDK with ciphers [TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_RSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_128_GCM_SHA256]
[2019-04-25T08:26:45,442][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslTransportServerProvider:JDK with ciphers [TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_RSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_128_GCM_SHA256]
[2019-04-25T08:26:45,442][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslHTTPProvider:JDK with ciphers [TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_RSA_WITH_AES_256_CBC_SHA, TLS_DHE_DSS_WITH_AES_256_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, TLS_DHE_RSA_WITH_AES_128_CBC_SHA, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, TLS_DHE_DSS_WITH_AES_128_GCM_SHA256]
[2019-04-25T08:26:45,443][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Enabled TLS protocols for transport layer : [TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,443][INFO ][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] Enabled TLS protocols for HTTP layer : [TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,443][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslTransportClientProvider:JDK with protocols [TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,443][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslTransportServerProvider:JDK with protocols [TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,443][DEBUG][c.f.s.s.DefaultSearchGuardKeyStore] [sgt-002] sslHTTPProvider:JDK with protocols [TLSv1.1, TLSv1.2]
[2019-04-25T08:26:45,655][INFO ][c.f.s.SearchGuardPlugin ] [sgt-002] Clustername: ashdev-elasticsearch
[2019-04-25T08:26:45,656][DEBUG][c.f.s.SearchGuardPlugin ] [sgt-002] This node [sgt-002] is a transportClient: false/tribeNode: false/tribeNodeClient: false
[2019-04-25T08:26:45,686][DEBUG][c.f.s.s.ReflectionHelper ] [sgt-002] Loaded module Module [type=DLSFLS, implementing class=com.floragunn.searchguard.configuration.SearchGuardFlsDlsIndexSearcherWrapper]
[2019-04-25T08:26:45,692][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [aggs-matrix-stats]
[2019-04-25T08:26:45,692][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [analysis-common]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [ingest-common]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [lang-expression]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [lang-mustache]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [lang-painless]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [mapper-extras]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [parent-join]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [percolator]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [rank-eval]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [reindex]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [repository-url]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [transport-netty4]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [tribe]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-ccr]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-core]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-deprecation]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-graph]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-logstash]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-ml]
[2019-04-25T08:26:45,693][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-monitoring]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-rollup]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-security]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-sql]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-upgrade]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded module [x-pack-watcher]
[2019-04-25T08:26:45,694][INFO ][o.e.p.PluginsService ] [sgt-002] loaded plugin [search-guard-6]
[2019-04-25T08:26:45,719][INFO ][c.f.s.SearchGuardPlugin ] [sgt-002] Disabled https compression by default to mitigate BREACH attacks. You can enable it by setting ‘http.compression: true’ in elasticsearch.yml
[2019-04-25T08:26:49,264][INFO ][o.e.x.m.j.p.l.CppLogMessageHandler] [sgt-002] [controller/4064] [Main.cc@109] controller (64 bit): Version 6.5.4 (Build b616085ef32393) Copyright (c) 2018 Elasticsearch BV
[2019-04-25T08:26:49,414][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Configured categories on rest layer to ignore: [AUTHENTICATED, GRANTED_PRIVILEGES]
[2019-04-25T08:26:49,414][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Configured categories on transport layer to ignore: [AUTHENTICATED, GRANTED_PRIVILEGES]
[2019-04-25T08:26:49,414][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Configured Users to ignore: [kibanaserver]
[2019-04-25T08:26:49,414][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Configured Users to ignore for read compliance events: [kibanaserver]
[2019-04-25T08:26:49,414][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Configured Users to ignore for write compliance events: [kibanaserver]
[2019-04-25T08:26:49,420][DEBUG][c.f.s.a.r.AsyncStoragePool] [sgt-002] Create new executor with threadPoolSize: 10 and maxQueueLen: 100000
[2019-04-25T08:26:49,421][INFO ][c.f.s.a.i.AuditLogImpl ] [sgt-002] Message routing enabled: true
[2019-04-25T08:26:49,421][DEBUG][c.f.s.a.i.AuditLogImpl ] [sgt-002] Security Manager present
[2019-04-25T08:26:49,422][DEBUG][c.f.s.a.i.AuditLogImpl ] [sgt-002] Shutdown Hook registered
[2019-04-25T08:26:49,422][DEBUG][c.f.s.s.ReflectionHelper ] [sgt-002] Loaded module Module [type=AUDITLOG, implementing class=com.floragunn.searchguard.auditlog.impl.AuditLogImpl]
[2019-04-25T08:26:49,423][WARN ][c.f.s.c.ComplianceConfig ] [sgt-002] If you plan to use field masking pls configure searchguard.compliance.salt to be a random string of 16 chars length identical on all nodes
[2019-04-25T08:26:49,424][INFO ][c.f.s.c.ComplianceConfig ] [sgt-002] PII configuration [auditLogPattern=org.joda.time.format.DateTimeFormatter@6b0ba697, auditLogIndex=null]: {}
[2019-04-25T08:26:49,442][DEBUG][c.f.s.SearchGuardPlugin ] [sgt-002] Compliance config is com.floragunn.searchguard.compliance.ComplianceConfig@1169fdfd because of dlsFlsAvailable: true and auditLog=class com.floragunn.searchguard.auditlog.impl.AuditLogImpl
[2019-04-25T08:26:49,443][DEBUG][c.f.s.SearchGuardPlugin ] [sgt-002] Using com.floragunn.searchguard.transport.DefaultInterClusterRequestEvaluator as intercluster request evaluator class
[2019-04-25T08:26:49,445][DEBUG][c.f.s.s.ReflectionHelper ] [sgt-002] Loaded module Module [type=MULTITENANCY, implementing class=com.floragunn.searchguard.configuration.PrivilegesInterceptorImpl]
[2019-04-25T08:26:49,449][DEBUG][c.f.s.c.AdminDNs ] [sgt-002] CN=kirk,OU=client,O=client,L=test, C=de is registered as an admin dn
[2019-04-25T08:26:49,450][DEBUG][c.f.s.c.AdminDNs ] [sgt-002] Loaded 1 admin DN’s [CN=kirk,OU=client,O=client,L=test, C=de]
[2019-04-25T08:26:49,451][DEBUG][c.f.s.c.AdminDNs ] [sgt-002] Loaded 0 impersonation DN’s {}
[2019-04-25T08:26:49,451][DEBUG][c.f.s.c.AdminDNs ] [sgt-002] Loaded 0 impersonation users for REST {}
[2019-04-25T08:26:49,454][DEBUG][c.f.s.c.ConfigurationLoader] [sgt-002] Index is: searchguard
[2019-04-25T08:26:49,455][DEBUG][c.f.s.c.LegacyConfigurationLoader] [sgt-002] Index is: searchguard
[2019-04-25T08:26:49,455][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type config with listener com.floragunn.searchguard.resolver.IndexResolverReplacer@c30f26d
[2019-04-25T08:26:49,456][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type config with listener com.floragunn.searchguard.http.XFFResolver@16944b58
[2019-04-25T08:26:49,464][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type config with listener com.floragunn.searchguard.auth.BackendRegistry@6487f7f8
[2019-04-25T08:26:49,471][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type roles with listener com.floragunn.searchguard.sgconf.ConfigModel@45f0038
[2019-04-25T08:26:49,471][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type rolesmapping with listener com.floragunn.searchguard.privileges.PrivilegesEvaluator@289a4b90
[2019-04-25T08:26:49,474][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type roles with listener com.floragunn.searchguard.privileges.PrivilegesEvaluator$TenantHolder@5f68eec6
[2019-04-25T08:26:49,475][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type config with listener com.floragunn.searchguard.configuration.CompatConfig@6cb194f5
[2019-04-25T08:26:49,478][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Subscribe on configuration changes by type config with listener com.floragunn.searchguard.SearchGuardPlugin$8@3dc2f14
[2019-04-25T08:26:49,664][DEBUG][o.e.a.ActionModule ] [sgt-002] Using REST wrapper from plugin com.floragunn.searchguard.SearchGuardPlugin
[2019-04-25T08:26:49,681][DEBUG][c.f.s.h.SearchGuardHttpServerTransport] [sgt-002] using max_chunk_size[8kb], max_header_size[8kb], max_initial_line_length[4kb], max_content_length[100mb], receive_predictor[64kb->64kb], max_composite_buffer_components[69905], pipelining[true], pipelining_max_events[10000]
[2019-04-25T08:26:49,996][INFO ][o.e.d.DiscoveryModule ] [sgt-002] using discovery type [zen] and host providers [settings]
[2019-04-25T08:26:50,663][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,663][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,664][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,664][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,665][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,665][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,666][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,666][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,667][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,667][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,668][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,668][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,669][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,669][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,670][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,670][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,670][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,671][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,671][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] Globally disabled endpoints: {}
[2019-04-25T08:26:50,671][DEBUG][c.f.s.d.r.a.RestApiPrivilegesEvaluator] [sgt-002] No disabled endpoints/methods for permitted role sg_all_access found, allowing all
[2019-04-25T08:26:50,672][DEBUG][c.f.s.s.ReflectionHelper ] [sgt-002] Loaded module Module [type=REST_MANAGEMENT_API, implementing class=com.floragunn.searchguard.dlic.rest.api.SearchGuardRestApiActions]
[2019-04-25T08:26:50,672][DEBUG][c.f.s.SearchGuardPlugin ] [sgt-002] Added 10 management rest handler(s)
[2019-04-25T08:26:50,672][INFO ][o.e.n.Node ] [sgt-002] initialized
[2019-04-25T08:26:50,672][INFO ][o.e.n.Node ] [sgt-002] starting …
[2019-04-25T08:26:50,738][DEBUG][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] using profile[default], worker_count[8], port[9300-9400], bind_host[[0.0.0.0]], publish_host[], compress[false], receive_predictor[64kb->64kb]
[2019-04-25T08:26:50,744][DEBUG][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] binding server bootstrap to: [0.0.0.0]
[2019-04-25T08:26:50,790][DEBUG][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] Bound profile [default] to address {0.0.0.0:9300}
[2019-04-25T08:26:50,791][INFO ][o.e.t.TransportService ] [sgt-002] publish_address {10.107.122.82:9300}, bound_addresses {0.0.0.0:9300}
[2019-04-25T08:26:50,829][INFO ][o.e.b.BootstrapChecks ] [sgt-002] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2019-04-25T08:26:50,841][INFO ][c.f.s.c.IndexBaseConfigurationRepository] [sgt-002] Check if searchguard index exists …
[2019-04-25T08:26:50,847][DEBUG][o.e.a.a.i.e.i.TransportIndicesExistsAction] [sgt-002] no known master node, scheduling a retry
[2019-04-25T08:26:51,017][WARN ][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] send message failed [channel: NettyTcpChannel{localAddress=0.0.0.0/0.0.0.0:45048, remoteAddress=/10.107.122.81:9300}]
java.nio.channels.ClosedChannelException: null
at io.netty.handler.ssl.SslHandler.channelInactive(…)(Unknown Source) ~[?:?]
[2019-04-25T08:26:51,017][WARN ][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] send message failed [channel: NettyTcpChannel{localAddress=0.0.0.0/0.0.0.0:34658, remoteAddress=/10.107.122.83:9300}]
java.nio.channels.ClosedChannelException: null
at io.netty.handler.ssl.SslHandler.channelInactive(…)(Unknown Source) ~[?:?]
[2019-04-25T08:26:51,946][WARN ][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] send message failed [channel: NettyTcpChannel{localAddress=0.0.0.0/0.0.0.0:45050, remoteAddress=/10.107.122.81:9300}]
java.nio.channels.ClosedChannelException: null
at io.netty.handler.ssl.SslHandler.channelInactive(…)(Unknown Source) ~[?:?]
[2019-04-25T08:26:51,959][WARN ][c.f.s.s.t.SearchGuardSSLNettyTransport] [sgt-002] send message failed [channel: NettyTcpChannel{localAddress=0.0.0.0/0.0.0.0:34664, remoteAddress=/10.107.122.83:9300}]
java.nio.channels.ClosedChannelException: null
ES config:
cluster.name: ashdev-elasticsearch
node.name: sgt-002
path.data: “/var/lib/elasticsearch”
path.logs: “/var/log/elasticsearch”
node.master: true
node.data: true
network.host: 0.0.0.0
discovery.zen.ping.unicast.hosts:
-
10.107.122.81
-
10.107.122.82
-
10.107.122.83
discovery.zen.minimum_master_nodes: 2
######## Start Search Guard Demo Configuration ########
WARNING: revise all the lines below before you go into production
searchguard.ssl.transport.pemcert_filepath: esnode.pem
searchguard.ssl.transport.pemkey_filepath: esnode-key.pem
searchguard.ssl.transport.pemtrustedcas_filepath: root-ca.pem
searchguard.ssl.transport.enforce_hostname_verification: false
searchguard.ssl.http.enabled: true
searchguard.ssl.http.pemcert_filepath: esnode.pem
searchguard.ssl.http.pemkey_filepath: esnode-key.pem
searchguard.ssl.http.pemtrustedcas_filepath: root-ca.pem
searchguard.allow_unsafe_democertificates: true
searchguard.allow_default_init_sgindex: true
searchguard.authcz.admin_dn:
- CN=kirk,OU=client,O=client,L=test, C=de
searchguard.audit.type: internal_elasticsearch
searchguard.enable_snapshot_restore_privilege: true
searchguard.check_snapshot_restore_write_privileges: true
searchguard.restapi.roles_enabled: [“sg_all_access”]
cluster.routing.allocation.disk.threshold_enabled: false
node.max_local_storage_nodes: 3
xpack.security.enabled: false
######## End Search Guard Demo Configuration ########
SearchGard conf:
searchguard:
dynamic:
http:
anonymous_auth_enabled: false
xff:
enabled: false
remoteIpHeader: ‘x-forwarded-for’
proxiesHeader: ‘x-forwarded-by’
authc:
kerberos_auth_domain:
http_enabled: false
transport_enabled: false
order: 6
http_authenticator:
challenge: true
config:
krb_debug: false
strip_realm_from_principal: true
authentication_backend:
type: noop
basic_internal_auth_domain:
http_enabled: true
transport_enabled: true
order: 4
http_authenticator:
type: basic
challenge: true
authentication_backend:
type: intern
proxy_auth_domain:
http_enabled: false
transport_enabled: false
order: 3
http_authenticator:
type: proxy
challenge: false
config:
user_header: “x-proxy-user”
roles_header: “x-proxy-roles”
authentication_backend:
type: noop
jwt_auth_domain:
http_enabled: false
transport_enabled: false
order: 0
http_authenticator:
type: jwt
challenge: false
config:
signing_key: “base64 encoded HMAC key or public RSA/ECDSA pem key”
jwt_header: “Authorization”
jwt_url_parameter: null
roles_key: null
subject_key: null
authentication_backend:
type: noop
clientcert_auth_domain:
http_enabled: false
transport_enabled: false
order: 2
http_authenticator:
type: clientcert
config:
challenge: false
authentication_backend:
type: noop
ldap:
http_enabled: false
transport_enabled: false
order: 5
http_authenticator:
type: basic
challenge: false
authentication_backend:
config:
enable_ssl: false
enable_start_tls: false
enable_ssl_client_auth: false
verify_hostnames: true
hosts:
- localhost:8389
bind_dn: null
password: null
userbase: ‘ou=people,dc=example,dc=com’
usersearch: ‘(sAMAccountName={0})’
username_attribute: null
authz:
roles_from_myldap:
http_enabled: false
transport_enabled: false
authorization_backend:
config:
enable_ssl: false
enable_start_tls: false
enable_ssl_client_auth: false
verify_hostnames: true
hosts:
- localhost:8389
bind_dn: null
password: null
rolebase: ‘ou=groups,dc=example,dc=com’
rolesearch: ‘(member={0})’
userroleattribute: null
userrolename: disabled
rolename: cn
resolve_nested_roles: true
userbase: ‘ou=people,dc=example,dc=com’
usersearch: ‘(uid={0})’
roles_from_another_ldap:
enabled: false
authorization_backend:
For TLS, the only important thing is that the chain is complete when comparing the certificates.