Kibana multitenancy plugin - unauthenticated access to /api/status results in AuthenticationError

Search Guard
1

Our current Kibana environment uses load balancers to distribute traffic and for high availability. It’s currently configured to check the Kibana status API endpoint to determine health of each node eg, https://10.10.0.1:5601/api/status. This configuration works, the load balancer can access this endpoint unauthenticated but Kibana will always log an multi-tenancy plugin AuthenticationError.

Elasticsearch and Search Guard is is working fine otherwise.

Any assistance is appreciated.

Thanks!

Elasticsearch version:
7.17.1 / search-guard-suite-plugin-7.17.1-53.0.0.zip

Server OS version:
RHEL 8

Kibana version (if relevant):
7.17.1 / search-guard-kibana-plugin-7.17.1-53.0.0.zip

Provide logs:
Kibana:

{"type":"log","@timestamp":"2022-03-30T13:01:36-00:00","tags":["warning","plugins","alerting"],"pid":69386,"message":"Error executing alerting apiKey invalidation task: Unauthorized"}
{"type":"log","@timestamp":"2022-03-30T13:01:38-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":69386,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"log","@timestamp":"2022-03-30T13:01:40-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":69386,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"log","@timestamp":"2022-03-30T13:01:42-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":69386,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"log","@timestamp":"2022-03-30T13:01:44-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":69386,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
2

So, to make sure I understand it correctly: The “Unauthorized” log messages appear to be triggered by the https://10.10.0.1:5601/api/status API call? Or could these be triggered by unrelated async processes?

3

Yes, it appears to be triggered by the /api/status call. I tested this in a test environment with no cluster or end-user activity. The only service that touched any part of the Kibana/Elasticsearch stack is the pair of load balancers checking the health of the Kibana node.

Thanks.

{"type":"response","@timestamp":"2022-03-31T02:13:34-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.252"},"res":{"statusCode":200,"responseTime":6,"contentLength":23585},"message":"GET /api/status 200 6ms - 23.0KB"}
{"type":"log","@timestamp":"2022-03-31T02:13:34-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":71063,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"response","@timestamp":"2022-03-31T02:13:34-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.253"},"res":{"statusCode":200,"responseTime":18,"contentLength":23585},"message":"GET /api/status 200 18ms - 23.0KB"}
{"type":"log","@timestamp":"2022-03-31T02:13:36-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":71063,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"response","@timestamp":"2022-03-31T02:13:36-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.252"},"res":{"statusCode":200,"responseTime":5,"contentLength":23585},"message":"GET /api/status 200 5ms - 23.0KB"}
{"type":"log","@timestamp":"2022-03-31T02:13:36-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":71063,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"response","@timestamp":"2022-03-31T02:13:36-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.253"},"res":{"statusCode":200,"responseTime":8,"contentLength":23585},"message":"GET /api/status 200 8ms - 23.0KB"}
{"type":"log","@timestamp":"2022-03-31T02:13:38-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":71063,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"response","@timestamp":"2022-03-31T02:13:38-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.252"},"res":{"statusCode":200,"responseTime":5,"contentLength":23600},"message":"GET /api/status 200 5ms - 23.0KB"}
{"type":"log","@timestamp":"2022-03-31T02:13:38-00:00","tags":["error","plugins","searchguard","searchguard-multitenancy"],"pid":71063,"message":"Multitenancy: Could not get authinfo AuthenticationError: Unauthorized"}
{"type":"response","@timestamp":"2022-03-31T02:13:38-00:00","tags":["api"],"pid":71063,"method":"get","statusCode":200,"req":{"url":"/api/status","method":"get","headers":{"connection":"close"},"remoteAddress":"10.10.0.253"},"res":{"statusCode":200,"responseTime":4,"contentLength":23600},"message":"GET /api/status 200 4ms - 23.0KB"}
4

I should also mention that this issue is logged by the audit and compliance feature of Search Guard which isn’t ideal…

Bump.

Thanks.

{
  "_index": "audit-000001",
  "_type": "_doc",
  "_id": "qrDRJIABBTgKSp6ztlRf",
  "_version": 1,
  "_score": 1,
  "_source": {
    "audit_cluster_name": "test-cluster",
    "audit_node_name": "coordinator1",
    "audit_category": "FAILED_LOGIN",
    "audit_request_origin": "REST",
    "audit_node_id": "gseuFD3NRIWSP7-jAawR1A",
    "audit_request_layer": "REST",
    "audit_rest_request_path": "/_searchguard/authinfo",
    "@timestamp": "2022-04-13T21:26:05.154+00:00",
    "audit_request_effective_user_is_admin": false,
    "audit_format_version": 4,
    "audit_request_remote_address": "10.10.0.118",
    "audit_node_host_address": "10.10.0.54",
    "audit_rest_request_headers": {
      "x-elastic-product-origin": [
        "kibana"
      ],
      "content-length": [
        "0"
      ],
      "x-elastic-client-meta": [
        "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
      ],
      "Connection": [
        "keep-alive"
      ],
      "Host": [
        "10.10.0.54:9200"
      ],
      "user-agent": [
        "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
      ]
    },
    "audit_request_effective_user": "<NONE>",
    "audit_node_host_name": "10.10.0.54"
  },
  "fields": {
    "audit_rest_request_headers.user-agent": [
      "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
    ],
    "audit_cluster_name": [
      "test-cluster"
    ],
    "audit_rest_request_headers.x-elastic-product-origin": [
      "kibana"
    ],
    "audit_rest_request_headers.x-elastic-client-meta.keyword": [
      "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
    ],
    "audit_request_origin.keyword": [
      "REST"
    ],
    "audit_rest_request_headers.x-elastic-product-origin.keyword": [
      "kibana"
    ],
    "audit_node_id.keyword": [
      "gseuFD3NRIWSP7-jAawR1A"
    ],
    "audit_rest_request_headers.Host.keyword": [
      "10.10.0.54:9200"
    ],
    "audit_rest_request_path": [
      "/_searchguard/authinfo"
    ],
    "audit_request_effective_user_is_admin": [
      false
    ],
    "audit_rest_request_path.keyword": [
      "/_searchguard/authinfo"
    ],
    "audit_format_version": [
      4
    ],
    "audit_request_remote_address": [
      "10.10.0.118"
    ],
    "audit_request_remote_address.keyword": [
      "10.10.0.118"
    ],
    "audit_rest_request_headers.Connection": [
      "keep-alive"
    ],
    "audit_rest_request_headers.Connection.keyword": [
      "keep-alive"
    ],
    "audit_node_name.keyword": [
      "coordinator1"
    ],
    "audit_rest_request_headers.content-length.keyword": [
      "0"
    ],
    "audit_rest_request_headers.x-elastic-client-meta": [
      "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
    ],
    "audit_request_layer.keyword": [
      "REST"
    ],
    "audit_rest_request_headers.content-length": [
      "0"
    ],
    "audit_cluster_name.keyword": [
      "test-cluster"
    ],
    "audit_node_host_address.keyword": [
      "10.10.0.54"
    ],
    "audit_node_name": [
      "coordinator1"
    ],
    "audit_category": [
      "FAILED_LOGIN"
    ],
    "audit_node_host_name.keyword": [
      "10.10.0.54"
    ],
    "audit_request_effective_user.keyword": [
      "<NONE>"
    ],
    "audit_request_origin": [
      "REST"
    ],
    "audit_request_layer": [
      "REST"
    ],
    "audit_node_id": [
      "gseuFD3NRIWSP7-jAawR1A"
    ],
    "@timestamp": [
      "2022-04-13T21:26:05.154Z"
    ],
    "audit_category.keyword": [
      "FAILED_LOGIN"
    ],
    "audit_node_host_address": [
      "10.10.0.54"
    ],
    "audit_request_effective_user": [
      "<NONE>"
    ],
    "audit_rest_request_headers.user-agent.keyword": [
      "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
    ],
    "audit_rest_request_headers.Host": [
      "10.10.0.54:9200"
    ],
    "audit_node_host_name": [
      "10.10.0.54"
    ]
  }
}
{
  "_index": "audit-000001",
  "_type": "_doc",
  "_id": "qbDRJIABBTgKSp6zrlSW",
  "_version": 1,
  "_score": 1,
  "_source": {
    "audit_cluster_name": "test-cluster",
    "audit_node_name": "coordinator1",
    "audit_category": "FAILED_LOGIN",
    "audit_request_origin": "REST",
    "audit_node_id": "gseuFD3NRIWSP7-jAawR1A",
    "audit_request_layer": "REST",
    "audit_rest_request_path": "/_searchguard/authinfo",
    "@timestamp": "2022-04-13T21:26:03.161+00:00",
    "audit_request_effective_user_is_admin": false,
    "audit_format_version": 4,
    "audit_request_remote_address": "10.10.0.118",
    "audit_node_host_address": "10.10.0.54",
    "audit_rest_request_headers": {
      "x-elastic-product-origin": [
        "kibana"
      ],
      "content-length": [
        "0"
      ],
      "x-elastic-client-meta": [
        "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
      ],
      "Connection": [
        "keep-alive"
      ],
      "Host": [
        "10.10.0.54:9200"
      ],
      "user-agent": [
        "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
      ]
    },
    "audit_request_effective_user": "<NONE>",
    "audit_node_host_name": "10.10.0.54"
  },
  "fields": {
    "audit_rest_request_headers.user-agent": [
      "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
    ],
    "audit_cluster_name": [
      "test-cluster"
    ],
    "audit_rest_request_headers.x-elastic-product-origin": [
      "kibana"
    ],
    "audit_rest_request_headers.x-elastic-client-meta.keyword": [
      "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
    ],
    "audit_request_origin.keyword": [
      "REST"
    ],
    "audit_rest_request_headers.x-elastic-product-origin.keyword": [
      "kibana"
    ],
    "audit_node_id.keyword": [
      "gseuFD3NRIWSP7-jAawR1A"
    ],
    "audit_rest_request_headers.Host.keyword": [
      "10.10.0.54:9200"
    ],
    "audit_rest_request_path": [
      "/_searchguard/authinfo"
    ],
    "audit_request_effective_user_is_admin": [
      false
    ],
    "audit_rest_request_path.keyword": [
      "/_searchguard/authinfo"
    ],
    "audit_format_version": [
      4
    ],
    "audit_request_remote_address": [
      "10.10.0.118"
    ],
    "audit_request_remote_address.keyword": [
      "10.10.0.118"
    ],
    "audit_rest_request_headers.Connection": [
      "keep-alive"
    ],
    "audit_rest_request_headers.Connection.keyword": [
      "keep-alive"
    ],
    "audit_node_name.keyword": [
      "coordinator1"
    ],
    "audit_rest_request_headers.content-length.keyword": [
      "0"
    ],
    "audit_rest_request_headers.x-elastic-client-meta": [
      "es=7.16.0p,js=16.13.2,t=7.16.0p,hc=16.13.2"
    ],
    "audit_request_layer.keyword": [
      "REST"
    ],
    "audit_rest_request_headers.content-length": [
      "0"
    ],
    "audit_cluster_name.keyword": [
      "test-cluster"
    ],
    "audit_node_host_address.keyword": [
      "10.10.0.54"
    ],
    "audit_node_name": [
      "coordinator1"
    ],
    "audit_category": [
      "FAILED_LOGIN"
    ],
    "audit_node_host_name.keyword": [
      "10.10.0.54"
    ],
    "audit_request_effective_user.keyword": [
      "<NONE>"
    ],
    "audit_request_origin": [
      "REST"
    ],
    "audit_request_layer": [
      "REST"
    ],
    "audit_node_id": [
      "gseuFD3NRIWSP7-jAawR1A"
    ],
    "@timestamp": [
      "2022-04-13T21:26:03.161Z"
    ],
    "audit_category.keyword": [
      "FAILED_LOGIN"
    ],
    "audit_node_host_address": [
      "10.10.0.54"
    ],
    "audit_request_effective_user": [
      "<NONE>"
    ],
    "audit_rest_request_headers.user-agent.keyword": [
      "elasticsearch-js/7.16.0-canary.7 (linux 4.18.0-348.20.1.el8_5.x86_64-x64; Node.js v16.13.2)"
    ],
    "audit_rest_request_headers.Host": [
      "10.10.0.54:9200"
    ],
    "audit_node_host_name": [
      "10.10.0.54"
    ]
  }
}
5

This topic was automatically closed 21 days after the last reply. New replies are no longer allowed.