Introduction
Hello,
I am attempting to setup an ES cluster inside docker with three nodes in both the master and data roles with one ES docker container acting as ingress node with search guard enabled. There is an issue when i enable to the mapped ports on my docker container as I have created a certificate for the native networking address and host, my question is how do I set my certificates up correctly so I can have this node act as the required ingress as the certificate work correctly when I disable the docker mapping or disable the HTTP interface, how can I find more debugging information on why the certificate becomes rejected as unknown?
Additional Information
- Search Guard and Elasticsearch version
Docker.elastic.co/elasticsearch/elasticsearch:6.2.2 & com.floragunn:search-guard-6:6.2.2-21.0
- Installed and used enterprise modules, if any
None
- JVM version and operating system version
Default one with the image
- Search Guard configuration files
Default at the time of testing
Docker-compose
version: ‘3’
services:
elasticsearch-ingress1:
container_name: es-g1
hostname: es-g1
restart: always
build:
context: ./elasticsearch/es-g1
environment:
-
bootstrap.memory_lock=true
-
“ES_JAVA_OPTS=-Xms512m -Xmx512m”
-
NODE_TLS_REJECT_UNAUTHORIZED=0
env_file:
- ./elasticsearch/es-g1/config.env
ulimits:
memlock:
soft: -1
hard: -1
volumes:
-
./elasticsearch/es-g1/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
-
./elasticsearch/es-g1/data:/usr/share/elasticsearch/data
-
./elasticsearch/backup:/usr/share/elasticsearch/backup
-
./license:/usr/share/elasticsearch/license:ro
-
./elasticsearch/es-g1/certs:/usr/share/elasticsearch/config/certs/
command: [“elasticsearch”, “-Elogger.level=DEBUG”]
ports:
- “10.6.0.3:9200:9200”
network_mode: isolated
labels:
org.label-scheme.name: “es-g1”
elasticsearch-master1:
container_name: es1
hostname: es1
restart: always
build:
context: ./elasticsearch/es1
environment:
-
bootstrap.memory_lock=true
-
“ES_JAVA_OPTS=-Xms512m -Xmx512m”
env_file:
- ./elasticsearch/es1/config.env
ulimits:
memlock:
soft: -1
hard: -1
volumes:
-
./elasticsearch/es1/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
-
./license:/usr/share/elasticsearch/license:ro
-
./elasticsearch/es1/data:/usr/share/elasticsearch/data
-
./elasticsearch/backup:/usr/share/elasticsearch/backup
-
./elasticsearch/es1/certs:/usr/share/elasticsearch/config/certs/
network_mode: isolated
labels:
org.label-scheme.name: “es1”
Second Master Container
elasticsearch-master2:
container_name: es2
hostname: es2
restart: always
build:
context: ./elasticsearch/es2
environment:
-
bootstrap.memory_lock=true
-
“ES_JAVA_OPTS=-Xms512m -Xmx512m”
env_file:
- ./elasticsearch/es2/config.env
ulimits:
memlock:
soft: -1
hard: -1
volumes:
-
./elasticsearch/es2/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
-
./elasticsearch/es2/data:/usr/share/elasticsearch/data
-
./elasticsearch/backup:/usr/share/elasticsearch/backup
-
./license:/usr/share/elasticsearch/license:ro
-
./elasticsearch/es2/certs:/usr/share/elasticsearch/config/certs/
network_mode: isolated
labels:
org.label-scheme.name: “es2”
Thrid Master Container
elasticsearch-master3:
container_name: es3
hostname: es3
restart: always
build:
context: ./elasticsearch/es3
environment:
-
bootstrap.memory_lock=true
-
“ES_JAVA_OPTS=-Xms512m -Xmx512m”
env_file:
- ./elasticsearch/es3/config.env
ulimits:
memlock:
soft: -1
hard: -1
volumes:
-
./elasticsearch/es3/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
-
./elasticsearch/es3/data:/usr/share/elasticsearch/data
-
./elasticsearch/backup:/usr/share/elasticsearch/backup
-
./license:/usr/share/elasticsearch/license:ro
-
./elasticsearch/es3/certs:/usr/share/elasticsearch/config/certs/
network_mode: isolated
labels:
org.label-scheme.name: “es3”
``
es-g1/elasticsearch.yml
Default Elasticsearch configuration from elasticsearch-docker.
from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml
cluster.name: “${CLUSTER_NAME}”
network.host: “${NETWORK_HOST}”
transport.publish_host: es-g1.isolated
Cluster discovery
discovery.zen.minimum_master_nodes: 2
discovery.zen.ping.unicast.hosts:
-
es1
-
es2
-
es3
#Node Placement Settings
node.master: false
node.data: false
node.ingest: true
search.remote.connect: false
#X Pack Settings
xpack.monitoring.enabled: “${XPACK_MONITORING_ENABLED}”
xpack.security.enabled: “${XPACK_SECURITY_ENABLED}”
#Search Guard Settings
searchguard.enterprise_modules_enabled: “${SEARCHG_EXTRA_MODULES_BOOL}”
searchguard.allow_unsafe_democertificates: “${SEARCHG_ALLOW_DEMO}”
searchguard.allow_default_init_sgindex: “${SEARCHG_ALLOW_DEFAULT_SGINDEX}”
searchguard.authcz.admin_dn:
- “${SEARCHG_AUTHCZ_ADMIN_DN}”
searchguard.nodes_dn:
- “CN=*,OU=operations,L=London,C=UK”
searchguard.ssl.transport.pemcert_filepath: “${SEARCHG_TRANSPORT_PEMCERT_LOC}”
searchguard.ssl.transport.pemkey_filepath: “${SEARCHG_TRANSPORT_PEMKEY_LOC}”
searchguard.ssl.transport.pemtrustedcas_filepath: “${SEARCHG_TRANSPORT_PEMCA_LOC}”
searchguard.ssl.transport.enforce_hostname_verification: “${SEARCHG_TRANSPORT_ENFORCE_HOST_VER}”
searchguard.ssl.transport.resolve_hostname: “${SEARCHG_TRANSPORT_RESOLVE_HOSTNAME}”
searchguard.ssl.transport.enabled_protocols:
- “TLSv1.2”
searchguard.ssl.http.enabled: “${SEARCHG_HTTP_ENABLED}”
searchguard.ssl.http.pemcert_filepath: “${SEARCHG_HTTP_PEMCERT_LOC}”
searchguard.ssl.http.pemkey_filepath: “${SEARCHG_HTTP_PEMKEY_LOC}”
searchguard.ssl.http.pemtrustedcas_filepath: “${SEARCHG_HTTP_PEMCA_LOC}”
searchguard.ssl.http.enabled_protocols:
- “TLSv1.2”
searchguard.audit.type: “${SEARCHG_AUDIT_TYPE}”
searchguard.enable_snapshot_restore_privilege: “${SEARCHG_ENABLE_SNAP_RESTORE_PRIV}”
searchguard.check_snapshot_restore_write_privileges: “${SEARCHG_CHECK_SNAP_RESTORE_WRITRE_PRIV}”
searchguard.restapi.roles_enabled: [“sg_all_access”]
``
config.env
CLUSTER_NAME=Armour-Production
NETWORK_HOST=0.0.0.0
XPACK_MONITORING_ENABLED=true
XPACK_SECURITY_ENABLED=false
#Search guard options
SEARCHG_EXTRA_MODULES_BOOL=false
SEARCHG_ALLOW_DEMO=false
SEARCHG_ALLOW_DEFAULT_SGINDEX=true
SEARCHG_AUTHCZ_ADMIN_DN=CN=*,OU=operations,L=London,C=UK
SEARCHG_AUDIT_TYPE=internal_elasticsearch
SEARCHG_ENABLE_SNAP_RESTORE_PRIV=true
SEARCHG_CHECK_SNAP_RESTORE_WRITRE_PRIV=true
#Search guard options transport interface
SEARCHG_TRANSPORT_PEMCERT_LOC=certs/transport.pem
SEARCHG_TRANSPORT_PEMKEY_LOC=certs/transport-key.pksc8.pem
SEARCHG_TRANSPORT_PEMCA_LOC=certs/armourcacert.pem
SEARCHG_TRANSPORT_ENFORCE_HOST_VER=false
SEARCHG_TRANSPORT_RESOLVE_HOSTNAME=true
#Search guard options http interface
SEARCHG_HTTP_ENABLED=true
SEARCHG_HTTP_PEMCERT_LOC=certs/es-g1.pem
SEARCHG_HTTP_PEMKEY_LOC=certs/es-g1-key.pksc8.pem
SEARCHG_HTTP_PEMCA_LOC=certs/armourcacert.pem
``
Transport.pem
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
35:10:4a:fa:e0:93:8c:5a:18:44:06:ab:26:5d:15:c9:ae:08:dd:bb
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=UK, ST=LONDON, <REDACTED>
Validity
Not Before: Jun 11 15:40:07 2018 GMT
Not After : Jul 13 15:40:31 2018 GMT
···
Subject: C=UK, L=London, OU=operations, CN=es-g1.isolated
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (4096 bit)
Modulus:
00:b3:3d:65:25:bd:8b:ae:6d:3d:a6:b6:9d:cb:71:
23:24:51:7e:a4:91:8d:08:77:99:90:76:ad:20:d0:
9c:72:98:35:57:15:2f:74:a4:58:26:ec:9d:c7:b0:
16:83:a5:a7:b8:9d:15:d4:f9:41:e8:51:95:a5:30:
06:f2:a0:a6:60:27:18:4d:4c:1c:2f:97:ee:23:c1:
0c:06:aa:40:46:86:42:df:05:d2:e3:0d:a2:75:4e:
3c:44:fb:8c:26:c4:84:a8:3a:07:b4:8e:cd:ff:c8:
50:d9:a4:63:0f:8a:5d:d2:0b:b4:2e:27:7c:c7:da:
85:74:54:a8:74:f1:d6:34:09:84:40:50:0a:8b:10:
e5:bd:ee:09:23:11:8f:76:59:c0:14:05:23:e9:53:
86:fb:20:3d:2f:04:b5:1a:f2:01:68:1f:63:8a:34:
be:8e:1a:b3:22:56:00:f1:68:29:ca:00:26:ab:ee:
4e:0e:ef:fb:10:23:2b:4c:a7:10:05:18:27:02:d8:
6c:a4:06:df:7f:c6:38:3a:dd:2a:62:1f:97:ce:ee:
c9:3d:ef:0a:fc:13:0f:2c:0e:75:20:28:d1:18:b1:
5b:e6:0f:54:30:00:36:4b:c0:12:a7:0d:f5:1d:f7:
fd:e8:f8:74:53:48:70:e8:7e:a1:99:83:26:84:82:
63:a2:19:40:e3:63:22:8c:58:a3:59:83:53:fe:ca:
55:ed:25:c1:d0:df:b6:91:6c:94:29:b9:0d:ca:09:
4f:f0:e4:85:c7:76:75:ec:73:28:31:ac:17:70:69:
1c:2f:ec:59:f4:c6:e4:c5:de:b0:dc:cf:20:3b:f7:
1f:a8:fe:92:1f:6b:ee:ad:ee:c3:a2:7b:a8:49:34:
a0:9e:df:f9:50:95:f8:f7:c5:4b:61:90:e6:a3:03:
39:5c:f8:c6:19:31:56:be:b9:b5:75:59:e5:78:71:
86:5f:be:ae:55:7d:45:8a:95:3f:12:89:53:1e:35:
38:1e:4c:5d:45:30:8d:e2:6f:39:64:d1:2c:8d:29:
89:48:e9:97:44:d4:1c:4d:ea:e6:ee:22:44:93:82:
9a:41:c4:c6:18:25:2c:44:78:c4:ab:39:88:f2:f2:
63:a8:13:29:6b:4d:84:38:44:24:84:de:76:82:dd:
b1:35:f8:b5:18:f7:41:81:74:2f:ee:da:d2:d3:cb:
54:b8:b4:24:5a:2d:cf:66:b5:68:9b:87:e3:77:f2:
fa:cd:84:f4:03:56:41:95:09:f1:69:73:dc:cd:25:
c3:a1:93:e2:14:02:8b:47:35:49:80:7a:2a:bf:08:
66:d0:d1:c6:40:89:b7:6e:cf:5a:ec:f6:99:ea:af:
7f:b8:8f
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Key Usage: critical
Digital Signature, Key Encipherment, Key Agreement
X509v3 Extended Key Usage:
TLS Web Server Authentication, TLS Web Client Authentication
X509v3 Subject Key Identifier:
06:05:F1:02:67:58:72:66:8D:C9:83:A7:B6:12:F9:A9:E4:14:BE:47
X509v3 Authority Key Identifier:
keyid:CB:BD:5D:64:0E:DA:1C:E2:67:30:51:7A:AE:56:BD:88:CA:9F:12:D8
X509v3 Subject Alternative Name:
DNS:es-g1.isolated
X509v3 CRL Distribution Points:
Full Name:
<REDACTED>
Signature Algorithm: sha256WithRSAEncryption
ab:d9:ef:90:3b:34:ff:a7:d3:32:24:9b:22:96:43:89:a5:59:
64:c9:56:8d:24:ed:9d:77:a6:e7:d0:9a:7b:ae:0c:cf:71:b7:
2e:45:f4:9e:7f:7b:7b:cd:dd:d0:ba:35:06:78:ba:ed:dc:1f:
1f:db:7e:dd:04:9b:bf:41:28:98:72:13:ec:bb:42:0c:34:f9:
2d:7a:ee:5e:03:6e:59:90:da:78:67:52:b4:cc:86:34:e8:6c:
66:03:49:16:63:3c:04:a0:16:18:03:f8:4d:86:9c:ed:2b:34:
80:7e:7f:42:67:d3:b7:2e:69:03:5b:d7:f4:e0:07:2b:6e:42:
7a:4a:98:36:07:b8:66:cd:62:44:3b:25:0b:91:7d:9c:15:36:
40:dc:4c:01:39:3b:ea:ac:37:08:5e:bf:9f:70:6f:a2:fa:dd:
99:96:09:a9:43:8b:cb:dd:d6:3e:f8:9c:8b:20:c0:b5:30:c3:
52:5c:06:c4:48:96:63:0f:a0:8b:81:9a:1f:f3:06:60:ad:95:
52:dc:57:df:27:dc:d9:1e:da:c3:76:a5:2f:75:5a:6f:35:62:
d5:00:7a:41:8f:0b:46:4f:a9:4f:c9:2f:4f:0e:3e:f8:90:53:
2e:49:4c:2c:c4:89:1e:b7:7b:2a:ae:0c:06:64:36:a6:e9:bd:
05:45:8f:52:99:c5:ed:96:8e:8d:2b:54:27:c4:6e:2e:93:a2:
f1:fd:d0:69:2d:d7:49:1e:05:e6:29:e4:b6:e8:17:7f:2d:24:
51:b1:a3:16:0c:42:f6:95:ad:99:06:98:76:12:13:2f:02:3a:
58:fd:b8:fb:60:ce:17:83:ab:12:7e:d2:d2:d2:e4:a2:a0:1f:
ab:93:da:6a:cd:01:f9:82:97:73:ab:2e:d1:d9:80:f8:81:a9:
17:95:5e:88:c1:a8:56:5c:e9:72:8a:7b:5b:07:67:28:fa:43:
f3:8e:81:c3:05:c6:ca:24:95:11:4b:82:d8:02:e5:ca:02:be:
bd:15:0b:8a:2f:5d:bd:d1:47:7d:0a:81:f4:fa:eb:9c:59:8c:
96:d4:d1:3d:b3:17:d9:e7:0b:8b:18:db:66:72:78:3b:1e:1a:
97:f0:d3:a2:8a:97:e2:96:08:d5:66:b9:b8:40:fa:dd:be:d5:
03:8a:89:37:d3:a5:07:fe:03:f1:e2:52:bb:38:ee:b5:6b:cf:
d6:2a:bc:ee:33:ed:a5:df:06:29:0a:bd:31:09:ea:16:37:46:
78:db:13:27:56:7d:5f:17:f6:8b:52:4a:82:b5:50:18:b9:ea:
02:14:1a:7b:80:07:37:37:24:9f:9d:50:c9:b7:14:00:2e:cb:
bb:8b:a8:19:34:04:07:20
``
es-g1.pem
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
22:e3:19:79:0f:6d:eb:8c:be:e7:1e:45:44:8b:39:c6:c1:d2:5b:62
Signature Algorithm: sha256WithRSAEncryption
Issuer: C=UK, ST=LONDON, O=
Validity
Not Before: Jun 11 15:42:23 2018 GMT
Not After : Jul 13 15:42:42 2018 GMT
Subject: C=UK, L=London, OU=operations, CN=elk.lon.
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (4096 bit)
Modulus:
00:b0:4e:14:6e:5e:5a:d1:b0:07:46:95:21:b0:ad:
cc:f5:62:6f:a0:de:de:6c:b0:4c:ec:a0:c8:2f:e5:
d5:7b:7b:51:d8:69:6a:06:94:77:6e:ba:10:49:43:
84:f7:ec:ea:65:b3:9f:38:fa:0d:95:31:a5:32:be:
74:be:12:0b:5c:68:e5:c0:45:90:06:79:76:eb:70:
10:a1:00:fd:bf:70:e3:a6:91:e3:a4:70:e0:76:da:
f4:b6:57:1e:17:45:a6:df:28:c0:af:5d:fa:b1:5d:
ac:fc:50:c4:65:e4:87:8e:74:7b:b9:13:40:0e:92:
4f:15:ec:2a:51:71:f7:50:f7:f1:f0:50:c8:9c:3e:
c2:ae:7f:3e:8a:15:86:90:a3:94:b9:36:ca:16:7a:
a4:37:12:5f:96:c9:49:13:85:ae:8f:99:cc:b6:aa:
fb:b6:0a:a3:99:b8:ff:5f:e9:94:53:a4:d6:e7:1e:
ae:d0:c7:f6:17:d3:a9:b8:b5:6f:76:7a:fe:aa:04:
9b:4e:ac:8b:f9:67:6a:e8:2e:cf:73:38:df:d1:ba:
57:05:89:28:a5:9d:f0:cf:20:67:99:4c:4d:6f:2f:
18:d5:1e:2b:2b:2a:9d:da:8b:a1:6e:8e:aa:d5:cd:
f6:06:01:c2:67:84:c3:3e:bf:fe:23:c1:83:9b:7f:
62:92:62:89:31:bb:f5:eb:d4:bb:15:c9:c6:af:3a:
78:59:7a:9b:ef:e9:96:e6:6e:e4:b4:c8:a0:de:22:
e5:37:c1:a7:46:9d:fe:31:9f:a8:df:14:99:3d:f9:
65:11:d7:43:66:3f:82:89:9e:84:3f:25:99:d8:fe:
61:b3:01:ec:da:8f:bd:2d:64:c4:86:47:60:38:e0:
83:e4:82:c8:d4:69:b4:9f:e3:7a:75:56:95:c6:80:
40:22:14:8a:35:34:3e:06:32:e2:59:f1:b4:a1:46:
c2:25:bb:55:66:ff:5b:04:28:b5:b7:a5:79:76:45:
b3:67:00:26:8c:9e:72:af:ed:cb:4d:ec:8f:b9:85:
59:d4:cd:38:9a:a2:cd:98:c9:e2:9a:04:94:4e:c0:
7c:3e:87:af:a3:80:4f:86:33:76:21:9f:e5:36:97:
3a:cd:29:90:01:25:55:a2:bf:e9:cb:d3:0f:09:1c:
c8:26:d9:54:98:12:e6:ff:71:0b:d5:69:b8:0e:a0:
e5:6e:d8:a2:1f:22:78:4c:a0:fe:3c:d2:a9:b2:ff:
24:3c:9f:05:09:6c:b7:e0:08:4e:b2:10:2a:e5:92:
b0:f4:f2:ab:0f:64:d8:70:f0:fd:ce:80:32:35:1b:
b1:0d:c1:37:ed:e6:4b:44:ed:aa:17:7a:88:d0:f9:
48:0d:03
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Key Usage: critical
Digital Signature, Key Encipherment, Key Agreement
X509v3 Extended Key Usage:
TLS Web Server Authentication, TLS Web Client Authentication
X509v3 Subject Key Identifier:
81:2F:3F:DE:C4:0D:85:33:2A:81:B4:71:86:C4:36:AB:AC:D8:83:E5
X509v3 Authority Key Identifier:
keyid:CB:BD:5D:64:0E:DA:1C:E2:67:30:51:7A:AE:56:BD:88:CA:9F:12:D8
X509v3 Subject Alternative Name:
DNS:elk.lon., DNS:es-g1.isolated, IP Address:10.6.0.3
X509v3 CRL Distribution Points:
Full Name:
Signature Algorithm: sha256WithRSAEncryption
7a:a0:54:35:db:a9:ed:c6:ba:05:c7:01:b4:f9:da:c9:fd:c5:
2e:d2:55:7c:67:bd:01:8a:80:50:d5:ca:82:9b:db:9d:43:ec:
d9:de:e6:bd:4b:bf:de:8d:e4:f6:aa:03:a8:d9:30:4d:3e:54:
1d:9e:d7:c2:f5:5f:e7:34:1f:a1:56:6c:5e:f6:95:ec:a6:96:
fe:ab:77:cc:43:bb:4c:59:0e:ab:63:06:bf:21:74:8b:c9:c1:
1f:82:b4:b2:87:4f:b5:25:75:4a:03:af:96:7d:63:62:15:6f:
75:5e:d3:23:f4:64:45:5c:7a:33:23:98:7c:ff:a9:56:d6:d5:
38:a1:00:ae:85:3a:67:da:79:9f:f3:c6:0a:b1:79:e8:73:c3:
d8:0e:d4:f1:40:3b:3e:1a:83:29:1b:43:da:e3:b1:26:c3:03:
59:e7:03:f3:d7:1c:13:fc:f3:2a:da:7d:04:27:f5:cd:ef:57:
27:f2:fe:3a:b5:46:fc:c4:11:68:e2:c2:fe:cd:b7:d6:27:af:
2d:fe:c6:2d:d3:a3:6c:b0:54:c9:4d:e9:08:51:61:95:83:f6:
8d:37:01:ec:56:0b:b6:51:f6:88:f9:f5:26:76:f5:21:f1:43:
ec:16:83:ce:e4:46:e5:e7:22:b5:ee:58:58:96:74:fb:2f:48:
5b:3c:43:99:ae:b3:97:0b:dc:cb:30:d4:32:b9:7a:19:d8:9e:
46:de:db:08:08:e0:83:9f:a3:1a:72:3f:17:05:37:95:e7:d7:
2d:d8:c2:1d:c2:16:a6:94:58:7e:b8:a2:9b:d8:4c:29:88:1a:
15:b1:69:43:96:5b:96:d9:ae:9d:ec:cc:dd:0b:f5:8d:1c:ee:
f6:91:67:f3:c6:9d:79:83:dd:e1:64:e9:ea:20:40:61:cf:8c:
30:61:2c:52:10:11:44:90:06:cf:52:e4:a7:7b:37:b7:27:2b:
9a:2a:75:60:d4:12:7b:de:ee:d5:a1:1f:43:f0:b6:eb:e8:f4:
02:a7:23:9d:cc:7b:9c:80:c9:9f:43:2a:22:b5:3b:ec:80:18:
e2:04:74:2e:fc:33:56:06:00:3a:c3:78:52:ef:58:92:0e:71:
34:a3:0e:a5:6a:70:ec:14:74:fa:5a:cc:88:db:bb:ed:7e:35:
45:93:72:94:09:7a:59:01:e8:c2:20:23:3b:dc:50:3b:3e:81:
f0:13:2c:2c:c6:c0:f0:6d:57:31:54:c1:c2:72:3b:90:94:1a:
ad:d0:45:f9:b6:b7:9e:9c:c1:f5:dc:fc:33:81:75:e4:5a:99:
be:4b:e5:aa:7f:b3:7c:94:bc:2d:e7:ca:00:16:97:75:1a:ac:
2e:74:e8:8a:b0:0a:09:9d
``
- Elasticsearch log messages on debug level
[2018-06-12T11:15:16,526][DEBUG][o.e.l.LicenseService ] [lPP4VTF] skipped license notifications reason: [1,state not recovered / initialized, blocks READ,WRITE,METADATA_READ,METADATA_WRITE]
[2018-06-12T11:15:16,526][DEBUG][o.e.x.m.e.l.LocalExporter] waiting until gateway has recovered from disk
[2018-06-12T11:15:16,526][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [2]])]: took [1.2s] done applying updated cluster state (version: 2, uuid: qr2HhQ03RNKNSl5mvsIU4Q)
[2018-06-12T11:15:16,529][DEBUG][i.n.h.s.SslHandler ] [id: 0xfd1ab45b, L:/172.18.0.5:52528 - R:es1.isolated/172.18.0.3:9300] HANDSHAKEN: TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384
[2018-06-12T11:15:16,568][DEBUG][i.n.h.s.SslHandler ] [id: 0xce98733f, L:/172.18.0.5:9300 - R:/172.18.0.3:40006] HANDSHAKEN: TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384
[2018-06-12T11:15:16,589][DEBUG][i.n.h.s.SslHandler ] [id: 0x9e666bcd, L:/172.18.0.5:9300 - R:/172.18.0.3:40004] HANDSHAKEN: TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384
[2018-06-12T11:15:16,816][DEBUG][o.e.d.z.PublishClusterStateAction] [lPP4VTF] received diff cluster state version [3] with uuid [yfAJchFTRWKLBriC9QsTXQ], diff size [13292]
[2018-06-12T11:15:16,827][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [3]])]: execute
[2018-06-12T11:15:16,827][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] cluster state updated, version [3], source [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [3]])]
[2018-06-12T11:15:16,835][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] applying cluster state version 3
[2018-06-12T11:15:16,835][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] apply cluster state with version 3
[2018-06-12T11:15:17,274][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] set locally applied cluster state to version 3
[2018-06-12T11:15:17,285][DEBUG][o.e.x.w.WatcherLifeCycleService] [lPP4VTF] not starting watcher. because the cluster isn’t ready yet to run watcher
[2018-06-12T11:15:17,304][DEBUG][o.e.l.LicenseService ] [lPP4VTF] previous [null]
[2018-06-12T11:15:17,325][DEBUG][o.e.l.LicenseService ] [lPP4VTF] current [LicensesMetaData{license={“uid”:“de13177c-c705-42dd-83b0-6a05664b34f7”,“type”:“trial”,“issue_date_in_millis”:1528796694456,“expiry_date_in_millis”:1531388694456,“max_nodes”:1000,“issued_to”:“Armour-Production”,“issuer”:“elasticsearch”,“signature”:“/////QAAAPAreOzCPticO49n2KREblsprfUljoEm50DYvZ9zXLRNO37EvNDoNWW+5dGqejl6yFI074GopHnZNWoR80gyrvZlbXCxzq8YTt+zbs+ld5OxObio+5bGQNuspBErBUESty8CRckhdQyg1VdOKtdCX6sSHeF6S6K3GvtJQ7v/oYI/eivCt48f/Z8i1nTSU+jcTG6itiioE8Of+S/U17Iy9J24HJ8vVgkKoy5F7hJIIVbt+mRqisMYAHUIXA/7iCZQpeMlyLLK9W3KZ8dE7LLRjctMDSA98GVgsV3uH+ATBtbjPu5Npo1sdP+b5uwko38EC7M=”,“start_date_in_millis”:-1}, trialVersion=6.2.2}]
[2018-06-12T11:15:17,346][INFO ][o.e.l.LicenseService ] [lPP4VTF] license [de13177c-c705-42dd-83b0-6a05664b34f7] mode [trial] - valid
[2018-06-12T11:15:17,351][DEBUG][o.e.l.LicenseService ] [lPP4VTF] license [de13177c-c705-42dd-83b0-6a05664b34f7] - valid
[2018-06-12T11:15:17,465][DEBUG][o.e.x.m.e.l.LocalExporter] started
[2018-06-12T11:15:17,491][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [3]])]: took [664ms] done applying updated cluster state (version: 3, uuid: yfAJchFTRWKLBriC9QsTXQ)
[2018-06-12T11:15:18,512][DEBUG][c.f.s.c.IndexBaseConfigurationRepository] Node started, try to initialize it. Wait for at least yellow cluster state…
[2018-06-12T11:15:18,550][DEBUG][o.e.d.z.PublishClusterStateAction] [lPP4VTF] received diff cluster state version [4] with uuid [kGRGWG2WQJeRstbqT00qPA], diff size [977]
[2018-06-12T11:15:18,566][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [4]])]: execute
[2018-06-12T11:15:18,566][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] cluster state updated, version [4], source [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [4]])]
[2018-06-12T11:15:18,566][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] applying cluster state version 4
[2018-06-12T11:15:18,566][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] apply cluster state with version 4
[2018-06-12T11:15:18,567][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] set locally applied cluster state to version 4
[2018-06-12T11:15:18,569][DEBUG][o.e.l.LicenseService ] [lPP4VTF] previous [LicensesMetaData{license={“uid”:“de13177c-c705-42dd-83b0-6a05664b34f7”,“type”:“trial”,“issue_date_in_millis”:1528796694456,“expiry_date_in_millis”:1531388694456,“max_nodes”:1000,“issued_to”:“Armour-Production”,“issuer”:“elasticsearch”,“signature”:“/////QAAAPAreOzCPticO49n2KREblsprfUljoEm50DYvZ9zXLRNO37EvNDoNWW+5dGqejl6yFI074GopHnZNWoR80gyrvZlbXCxzq8YTt+zbs+ld5OxObio+5bGQNuspBErBUESty8CRckhdQyg1VdOKtdCX6sSHeF6S6K3GvtJQ7v/oYI/eivCt48f/Z8i1nTSU+jcTG6itiioE8Of+S/U17Iy9J24HJ8vVgkKoy5F7hJIIVbt+mRqisMYAHUIXA/7iCZQpeMlyLLK9W3KZ8dE7LLRjctMDSA98GVgsV3uH+ATBtbjPu5Npo1sdP+b5uwko38EC7M=”,“start_date_in_millis”:-1}, trialVersion=6.2.2}]
[2018-06-12T11:15:18,569][DEBUG][o.e.l.LicenseService ] [lPP4VTF] current [LicensesMetaData{license={“uid”:“de13177c-c705-42dd-83b0-6a05664b34f7”,“type”:“trial”,“issue_date_in_millis”:1528796694456,“expiry_date_in_millis”:1531388694456,“max_nodes”:1000,“issued_to”:“Armour-Production”,“issuer”:“elasticsearch”,“signature”:“/////QAAAPAreOzCPticO49n2KREblsprfUljoEm50DYvZ9zXLRNO37EvNDoNWW+5dGqejl6yFI074GopHnZNWoR80gyrvZlbXCxzq8YTt+zbs+ld5OxObio+5bGQNuspBErBUESty8CRckhdQyg1VdOKtdCX6sSHeF6S6K3GvtJQ7v/oYI/eivCt48f/Z8i1nTSU+jcTG6itiioE8Of+S/U17Iy9J24HJ8vVgkKoy5F7hJIIVbt+mRqisMYAHUIXA/7iCZQpeMlyLLK9W3KZ8dE7LLRjctMDSA98GVgsV3uH+ATBtbjPu5Npo1sdP+b5uwko38EC7M=”,“start_date_in_millis”:-1}, trialVersion=6.2.2}]
[2018-06-12T11:15:18,569][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [4]])]: took [3ms] done applying updated cluster state (version: 4, uuid: kGRGWG2WQJeRstbqT00qPA)
[2018-06-12T11:15:18,570][DEBUG][o.e.x.w.WatcherLifeCycleService] [lPP4VTF] not starting watcher. because the cluster isn’t ready yet to run watcher
[2018-06-12T11:15:18,731][DEBUG][o.e.m.o.OsProbe ] error reading control group stats
java.nio.file.NoSuchFileException: /sys/fs/cgroup/cpuacct/docker/49a5a242112046328ba0811d40fc89d1c3a82394117e7e3a044000ac4954ae9e/cpuacct.usage
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_161]
at java.nio.file.Files.newByteChannel(Files.java:407) ~[?:1.8.0_161]
at java.nio.file.spi.FileSystemProvider.newInputStream(FileSystemProvider.java:384) ~[?:1.8.0_161]
at java.nio.file.Files.newInputStream(Files.java:152) ~[?:1.8.0_161]
at java.nio.file.Files.newBufferedReader(Files.java:2784) ~[?:1.8.0_161]
at java.nio.file.Files.readAllLines(Files.java:3202) ~[?:1.8.0_161]
at java.nio.file.Files.readAllLines(Files.java:3242) ~[?:1.8.0_161]
at org.elasticsearch.monitor.os.OsProbe.readSingleLine(OsProbe.java:183) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.readSysFsCgroupCpuAcctCpuAcctUsage(OsProbe.java:272) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.getCgroupCpuAcctUsageNanos(OsProbe.java:259) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.getCgroup(OsProbe.java:478) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.osStats(OsProbe.java:531) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:64) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:57) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService.stats(OsService.java:54) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.node.NodeService.stats(NodeService.java:113) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:74) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.admin.cluster.node.stats.TransportNodesStatsAction.nodeOperation(TransportNodesStatsAction.java:39) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:262) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:258) [elasticsearch-6.2.2.jar:6.2.2]
at com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceivedDecorate(SearchGuardSSLRequestHandler.java:170) [search-guard-ssl-6.2.2-25.1.jar:6.2.2-25.1]
at com.floragunn.searchguard.transport.SearchGuardRequestHandler.messageReceivedDecorate(SearchGuardRequestHandler.java:121) [search-guard-6-6.2.2-21.0.jar:6.2.2-21.0]
at com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceived(SearchGuardSSLRequestHandler.java:92) [search-guard-ssl-6.2.2-25.1.jar:6.2.2-25.1]
at com.floragunn.searchguard.SearchGuardPlugin$6$1.messageReceived(SearchGuardPlugin.java:526) [search-guard-6-6.2.2-21.0.jar:6.2.2-21.0]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.transport.TransportService$7.doRun(TransportService.java:656) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.2.2.jar:6.2.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
[2018-06-12T11:15:20,175][DEBUG][o.e.m.o.OsProbe ] error reading control group stats
java.nio.file.NoSuchFileException: /sys/fs/cgroup/cpuacct/docker/49a5a242112046328ba0811d40fc89d1c3a82394117e7e3a044000ac4954ae9e/cpuacct.usage
at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) ~[?:?]
at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) ~[?:?]
at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) ~[?:?]
at java.nio.file.Files.newByteChannel(Files.java:361) ~[?:1.8.0_161]
at java.nio.file.Files.newByteChannel(Files.java:407) ~[?:1.8.0_161]
at java.nio.file.spi.FileSystemProvider.newInputStream(FileSystemProvider.java:384) ~[?:1.8.0_161]
at java.nio.file.Files.newInputStream(Files.java:152) ~[?:1.8.0_161]
at java.nio.file.Files.newBufferedReader(Files.java:2784) ~[?:1.8.0_161]
at java.nio.file.Files.readAllLines(Files.java:3202) ~[?:1.8.0_161]
at java.nio.file.Files.readAllLines(Files.java:3242) ~[?:1.8.0_161]
at org.elasticsearch.monitor.os.OsProbe.readSingleLine(OsProbe.java:183) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.readSysFsCgroupCpuAcctCpuAcctUsage(OsProbe.java:272) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.getCgroupCpuAcctUsageNanos(OsProbe.java:259) ~[elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.getCgroup(OsProbe.java:478) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsProbe.osStats(OsProbe.java:531) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:64) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService$OsStatsCache.refresh(OsService.java:57) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.SingleObjectCache.getOrRefresh(SingleObjectCache.java:54) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.monitor.os.OsService.stats(OsService.java:54) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.node.NodeService.stats(NodeService.java:113) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:95) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction.nodeOperation(TransportClusterStatsAction.java:50) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:140) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:262) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:258) [elasticsearch-6.2.2.jar:6.2.2]
at com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceivedDecorate(SearchGuardSSLRequestHandler.java:170) [search-guard-ssl-6.2.2-25.1.jar:6.2.2-25.1]
at com.floragunn.searchguard.transport.SearchGuardRequestHandler.messageReceivedDecorate(SearchGuardRequestHandler.java:235) [search-guard-6-6.2.2-21.0.jar:6.2.2-21.0]
at com.floragunn.searchguard.ssl.transport.SearchGuardSSLRequestHandler.messageReceived(SearchGuardSSLRequestHandler.java:142) [search-guard-ssl-6.2.2-25.1.jar:6.2.2-25.1]
at com.floragunn.searchguard.SearchGuardPlugin$6$1.messageReceived(SearchGuardPlugin.java:526) [search-guard-6-6.2.2-21.0.jar:6.2.2-21.0]
at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:66) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.transport.TcpTransport$RequestHandler.doRun(TcpTransport.java:1555) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingAbstractRunnable.doRun(ThreadContext.java:672) [elasticsearch-6.2.2.jar:6.2.2]
at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) [elasticsearch-6.2.2.jar:6.2.2]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
[2018-06-12T11:15:20,437][DEBUG][o.e.d.z.PublishClusterStateAction] [lPP4VTF] received diff cluster state version [5] with uuid [1-PoVmHqQ767yTrgSzsnjA], diff size [961]
[2018-06-12T11:15:20,444][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [5]])]: execute
[2018-06-12T11:15:20,444][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] cluster state updated, version [5], source [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [5]])]
[2018-06-12T11:15:20,452][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] applying cluster state version 5
[2018-06-12T11:15:20,452][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] apply cluster state with version 5
[2018-06-12T11:15:20,452][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] set locally applied cluster state to version 5
[2018-06-12T11:15:20,454][DEBUG][o.e.x.w.WatcherService ] [lPP4VTF] starting watch service…
[2018-06-12T11:15:20,457][DEBUG][o.e.x.w.e.ExecutionService] [lPP4VTF] starting execution service
[2018-06-12T11:15:20,457][DEBUG][o.e.x.w.e.ExecutionService] [lPP4VTF] started execution service
[2018-06-12T11:15:20,463][DEBUG][o.e.l.LicenseService ] [lPP4VTF] previous [LicensesMetaData{license={“uid”:“de13177c-c705-42dd-83b0-6a05664b34f7”,“type”:“trial”,“issue_date_in_millis”:1528796694456,“expiry_date_in_millis”:1531388694456,“max_nodes”:1000,“issued_to”:“Armour-Production”,“issuer”:“elasticsearch”,“signature”:“/////QAAAPAreOzCPticO49n2KREblsprfUljoEm50DYvZ9zXLRNO37EvNDoNWW+5dGqejl6yFI074GopHnZNWoR80gyrvZlbXCxzq8YTt+zbs+ld5OxObio+5bGQNuspBErBUESty8CRckhdQyg1VdOKtdCX6sSHeF6S6K3GvtJQ7v/oYI/eivCt48f/Z8i1nTSU+jcTG6itiioE8Of+S/U17Iy9J24HJ8vVgkKoy5F7hJIIVbt+mRqisMYAHUIXA/7iCZQpeMlyLLK9W3KZ8dE7LLRjctMDSA98GVgsV3uH+ATBtbjPu5Npo1sdP+b5uwko38EC7M=”,“start_date_in_millis”:-1}, trialVersion=6.2.2}]
[2018-06-12T11:15:20,463][DEBUG][o.e.l.LicenseService ] [lPP4VTF] current [LicensesMetaData{license={“uid”:“de13177c-c705-42dd-83b0-6a05664b34f7”,“type”:“trial”,“issue_date_in_millis”:1528796694456,“expiry_date_in_millis”:1531388694456,“max_nodes”:1000,“issued_to”:“Armour-Production”,“issuer”:“elasticsearch”,“signature”:“/////QAAAPAreOzCPticO49n2KREblsprfUljoEm50DYvZ9zXLRNO37EvNDoNWW+5dGqejl6yFI074GopHnZNWoR80gyrvZlbXCxzq8YTt+zbs+ld5OxObio+5bGQNuspBErBUESty8CRckhdQyg1VdOKtdCX6sSHeF6S6K3GvtJQ7v/oYI/eivCt48f/Z8i1nTSU+jcTG6itiioE8Of+S/U17Iy9J24HJ8vVgkKoy5F7hJIIVbt+mRqisMYAHUIXA/7iCZQpeMlyLLK9W3KZ8dE7LLRjctMDSA98GVgsV3uH+ATBtbjPu5Npo1sdP+b5uwko38EC7M=”,“start_date_in_millis”:-1}, trialVersion=6.2.2}]
[2018-06-12T11:15:20,469][DEBUG][o.e.c.s.ClusterApplierService] [lPP4VTF] processing [apply cluster state (from master [master {8hEVRRM}{8hEVRRMPRbSPo4LwtO20UQ}{VFHzoM1eRs-KztSC9tm3xQ}{es2.isolated}{172.18.0.4:9300}{ml.machine_memory=8357396480, ml.max_open_jobs=20, ml.enabled=true} committed version [5]])]: took [24ms] done applying updated cluster state (version: 5, uuid: 1-PoVmHqQ767yTrgSzsnjA)
[2018-06-12T11:15:20,909][DEBUG][o.e.x.w.e.ExecutionService] [lPP4VTF] triggered execution of [0] watches
[2018-06-12T11:15:20,909][DEBUG][o.e.x.w.WatcherService ] [lPP4VTF] watch service has started
[2018-06-12T11:15:21,097][ERROR][c.f.s.h.SearchGuardHttpServerTransport] [lPP4VTF] SSL Problem Received fatal alert: certificate_unknown
javax.net.ssl.SSLException: Received fatal alert: certificate_unknown
at sun.security.ssl.Alerts.getSSLException(Alerts.java:208) ~[?:?]
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1666) ~[?:?]
at sun.security.ssl.SSLEngineImpl.fatal(SSLEngineImpl.java:1634) ~[?:?]
at sun.security.ssl.SSLEngineImpl.recvAlert(SSLEngineImpl.java:1800) ~[?:?]
at sun.security.ssl.SSLEngineImpl.readRecord(SSLEngineImpl.java:1083) ~[?:?]
at sun.security.ssl.SSLEngineImpl.readNetRecord(SSLEngineImpl.java:907) ~[?:?]
at sun.security.ssl.SSLEngineImpl.unwrap(SSLEngineImpl.java:781) ~[?:?]
at javax.net.ssl.SSLEngine.unwrap(SSLEngine.java:624) ~[?:1.8.0_161]
at io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:281) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1215) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1127) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1162) ~[netty-handler-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265) ~[netty-codec-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:545) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459) [netty-transport-4.1.16.Final.jar:4.1.16.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.16.Final.jar:4.1.16.Final]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]
``