TL;DR
Getting HTTP-302 redirect error when starting the Kibana image in Kubernetes with Search Guard enabled for TLS communication with Elasticsearch, but not with users accessing via a web browser.
Details
Configuration
I’m using the following configuration:
- Elasticsearch and Kibana v7.6.2 images, from Elastic, with corresponding versions of Search Guard plugins installed.
- Elastic Helm Charts, v7.6.2.
- TLS enabled for communication between Kibana and Elasticsearch, but not for users connecting to Kibana (I think).
Issue
I am getting the following error from the Kibana readiness probe, which attempts to connect to http://localhost:5601/app/kibana in the Kibana log when starting the Kibana image with the Elastic Helm Charts, with the Kibana plugins installed and configured in both Elasticsearch and Kibana. Additionally, when I attempt to connect to Kibana with a web browser or curl, I am unable to.
The error is:
{"type":"response","@timestamp":"2020-04-23T16:41:17Z","tags":[],"pid":6,"method":"get","statusCode":302,"req":{"url":"/app/kibana","method":"get","headers":{"user-agent":"curl/7.29.0","host":"localhost:5601","accept":"*/*"},"remoteAddress":"127.0.0.1","userAgent":"127.0.0.1"},"res":{"statusCode":302,"responseTime":1,"contentLength":9},"message":"GET /app/kibana 302 1ms - 9.0B"}
This post reported the same issue, but I don’t appear to have the config settings set, which were at issue.
In this post I was experiencing issues getting Kibana and Elasticsearch to communicate via TLS. However, this issue appears to have been resolved, as can be seen when I connect to one of the logstash nodes and execute the following query:
curl -k -u $ELASTIC_USERNAME:$ELASTIC_PASSWORD -H 'Content-Type: application/json' \
https://localhost:9200/.monitoring-kibana-7-*/_search?pretty -d '
{
"query": {
"match_all": {}
},
"size": 1,
"sort": [
{
"timestamp": {
"order": "desc"
}
}
]
}'
I receive the following output, which appears to indicate that Kibana is communicating to Elasticsearch on an ongoing basis (see timestamp):
{
"took" : 58,
"timed_out" : false,
"_shards" : {
"total" : 7,
"successful" : 7,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 10000,
"relation" : "gte"
},
"max_score" : null,
"hits" : [
{
"_index" : ".monitoring-kibana-7-2020.04.23",
"_type" : "_doc",
"_id" : "tIvwp3EBjwb6QOtkyCo0",
"_score" : null,
"_source" : {
"cluster_uuid" : "hvkuxjblQ3WsHjz8ON35-g",
"timestamp" : "2020-04-23T16:48:27.185Z",
"interval_ms" : 10000,
"type" : "kibana_stats",
"source_node" : {
"uuid" : "urvAbq76SqCjnAEPsr_hyA",
"host" : "10.229.94.58",
"transport_address" : "10.229.94.58:9300",
"ip" : "10.229.94.58",
"name" : "elk-es-coord-0",
"timestamp" : "2020-04-23T16:48:27.185Z"
},
"kibana_stats" : {
"kibana" : {
"uuid" : "5ddddc36-822d-4c5e-aa2d-2aa1c883f025",
"name" : "elk-kibana-6b7d4d7dc6-cn9xj",
"index" : ".kibana",
"host" : "0.0.0.0",
"transport_address" : "0.0.0.0:5601",
"version" : "7.6.2",
"snapshot" : false,
"status" : "red"
},
"cloud" : {
"name" : "aws",
"id" : "i-000c8043efc0be372",
"vm_type" : "r5d.4xlarge",
"region" : "us-east-1",
"zone" : "us-east-1c",
"metadata" : {
"architecture" : "x86_64",
"marketplaceProductCodes" : null,
"imageId" : "ami-0dc7713312a7ec987",
"kernelId" : null,
"pendingTime" : "2020-04-02T18:17:30Z",
"ramdiskId" : null,
"version" : "2017-09-30"
}
},
"concurrent_connections" : 0,
"os" : {
"load" : {
"1m" : 3.01611328125,
"5m" : 2.9482421875,
"15m" : 2.87060546875
},
"memory" : {
"total_in_bytes" : 133653843968,
"free_in_bytes" : 120397815808,
"used_in_bytes" : 13256028160
},
"uptime_in_millis" : 747809000,
"platform" : "linux",
"platformRelease" : "linux-4.14.171-136.231.amzn2.x86_64",
"distro" : "Centos",
"distroRelease" : "Centos-7.7.1908"
},
"process" : {
"event_loop_delay" : 1.523249626159668,
"memory" : {
"heap" : {
"total_in_bytes" : 858374144,
"used_in_bytes" : 599807576,
"size_limit" : 2217857988
},
"resident_set_size_in_bytes" : 1243553792
},
"uptime_in_millis" : 9496096
},
"requests" : {
"disconnects" : 0,
"total" : 2
},
"response_times" : {
"average" : 2,
"max" : 2
},
"timestamp" : "2020-04-23T16:48:24.960Z"
}
},
"sort" : [
1587660507185
]
}
]
}
}
kibana.yml:
server:
xsrf:
whitelist:
- "/searchguard/saml/acs/idpinitiated"
- "/searchguard/saml/acs"
- "/searchguard/saml/logout"
# Xpack configuration.
xpack:
# Configure xpack security.
security.enabled: false
elasticsearch:
username: "${SG_KIBANASERVER_USER}"
password: "${SG_KIBANASERVER_PASSWD}"
ssl:
alwaysPresentCertificate: false
certificate: /usr/share/kibana/config/certs/node/cert.pem
key: /usr/share/kibana/config/certs/node/key.pem
certificateAuthorities:
- /usr/share/kibana/config/certs/node/ca_bundle.pem
verificationMode: none
elasticsearch.yml
Note that I’m reasonably certain that the issue is not on the Elasticsearch side, but I’m including this for
action:
auto_create_index: ".*,sg7-*,searchguard,searchguard*"
# Allow both primary and non-primary shards to be moved between nodes for
# balancing purposes.
cluster:
routing.allocation.enable: all
xpack:
# Disable xpack.security when using SearchGuard.
security.enabled: false
# Cluster monitoring configuration.
monitoring:
enabled: true
collection:
enabled: true
# SearchGuard configuration.
searchguard:
# Next line is for use in sandbox only.
# allow_unsafe_democertificates: true
nodes_dn:
- EMAILADDRESS=pki@example.com,CN=*.cluster.local,OU=Example Inc.,L=Smallville,ST=Kansas,C=US
allow_default_init_sgindex: true
audit:
type: internal_elasticsearch
authcz:
admin_dn:
- EMAILADDRESS=pki@example.com,CN=elk.example.com,OU=Example Inc.,L=Smallville,ST=Kansas,C=US
check_snapshot_restore_write_privileges: true
enable_snapshot_restore_privilege: true
restapi:
roles_enabled:
- SGS_ALL_ACCESS
# Searchguard config
ssl:
http:
enabled: true
pemcert_filepath: certs/node/cert.pem
pemkey_filepath: certs/node/key.p8
pemtrustedcas_filepath: certs/node/ca_bundle.pem
# clientauth_mode: OPTIONAL
clientauth_mode: NONE
enabled_protocols:
- TLSv1.3
- TLSv1.2
- TLSv1.1
transport:
# Mount certs to /usr/share/elasticsearch/config
pemcert_filepath: certs/node/cert.pem
pemkey_filepath: certs/node/key.p8
pemtrustedcas_filepath: certs/node/ca_bundle.pem
enforce_hostname_verification: false