Hi,
we’re using SG 2.0.20 in all of our ES 6.6.1 clusters, and we wanted to start using the X-Opaque-Id header to identify running tasks in the slow logs.
Unfortunately, with SG enabled, the header doesn’t seem to propagate to ES’s child tasks.
An example of this behavior would look like this:
- With SG disabled:
>>> curl -s -H "X-Opaque-Id:TEST" http://localhost:9200/_tasks?group_by=parents | jq
{
"tasks": {
"yWmK4i_XRs6WcrPRRUJPyg:790": {
"node": "yWmK4i_XRs6WcrPRRUJPyg",
"id": 790,
"type": "transport",
"action": "cluster:monitor/tasks/lists",
"start_time_in_millis": 1554367389882,
"running_time_in_nanos": 190362,
"cancellable": false,
"headers": {
"X-Opaque-Id": "TEST"
},
"children": [
{
"node": "T4S6DTEoTcCgcM9LmnpfeA",
"id": 5058,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 44156,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "yWmK4i_XRs6WcrPRRUJPyg",
"id": 791,
"type": "direct",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389882,
"running_time_in_nanos": 58913,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "5JVOoXP5Sf2WqvcrrsSoQw",
"id": 5470,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 52978,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "laBgJRpQS46evIlJ0ERFFw",
"id": 12518,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 52324,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "0MMcRoB8SsKObQaoPgepuA",
"id": 15406,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 50740,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "On4x4qcgTDepEK4RN_bi5A",
"id": 5463,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 50685,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
},
{
"node": "Ln2c3tFDTnqrOmSWz2mWjg",
"id": 23483,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367389883,
"running_time_in_nanos": 44646,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:790",
"headers": {
"X-Opaque-Id": "TEST"
}
}
]
}
}
}
- With SG enabled:
>>> curl -s -H "X-Opaque-Id:TEST" https://localhost:9200/_tasks?group_by=parents | jq
{
"tasks": {
"yWmK4i_XRs6WcrPRRUJPyg:348": {
"node": "yWmK4i_XRs6WcrPRRUJPyg",
"id": 348,
"type": "transport",
"action": "cluster:monitor/tasks/lists",
"start_time_in_millis": 1554367832916,
"running_time_in_nanos": 2477570,
"cancellable": false,
"headers": {
"X-Opaque-Id": "TEST"
},
"children": [
{
"node": "0MMcRoB8SsKObQaoPgepuA",
"id": 645,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832918,
"running_time_in_nanos": 512547,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "5JVOoXP5Sf2WqvcrrsSoQw",
"id": 167,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832918,
"running_time_in_nanos": 1686419,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "Ln2c3tFDTnqrOmSWz2mWjg",
"id": 597,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832918,
"running_time_in_nanos": 484345,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "laBgJRpQS46evIlJ0ERFFw",
"id": 437,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832919,
"running_time_in_nanos": 679146,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "T4S6DTEoTcCgcM9LmnpfeA",
"id": 422,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832919,
"running_time_in_nanos": 655382,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "yWmK4i_XRs6WcrPRRUJPyg",
"id": 349,
"type": "direct",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832917,
"running_time_in_nanos": 1131946,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
},
{
"node": "On4x4qcgTDepEK4RN_bi5A",
"id": 184,
"type": "netty",
"action": "cluster:monitor/tasks/lists[n]",
"start_time_in_millis": 1554367832919,
"running_time_in_nanos": 910877,
"cancellable": false,
"parent_task_id": "yWmK4i_XRs6WcrPRRUJPyg:348",
"headers": {}
}
]
}
}
}
As you can see, in both cases the parent task has the header, but only with SG disabled do the child tasks receive it.
I wasn’t able to find any documentation either on the ES or the SG side, that would allow me to either whitelist the header, or fix this issue in some other manner.
Here are the relevant parts from my elasticsearch.yml file:
searchguard.authcz.admin_dn:
- CN=admin, OU=client
searchguard.audit.type: internal_elasticsearch
searchguard.ssl.http.enable_openssl_if_available: true
searchguard.ssl.http.enabled: true
searchguard.ssl.http.keystore_filepath: "path_to_js.jks"
searchguard.ssl.http.keystore_password: ks_pass
searchguard.ssl.http.truststore_filepath: "path_to_ts.jks"
searchguard.ssl.http.truststore_password: ts_pass
searchguard.ssl.transport.enable_openssl_if_available: true
searchguard.ssl.transport.enforce_hostname_verification: false
searchguard.ssl.transport.keystore_filepath: "path_to_ks.jks"
searchguard.ssl.transport.keystore_password: ks_pass
searchguard.ssl.transport.truststore_filepath: "path_to_ts.jks"
searchguard.ssl.transport.truststore_password: ts_pass
searchguard.enterprise_modules_enabled: false
searchguard.enable_snapshot_restore_privilege: true
I haven’t seen anything in the ES when running the API call.
This is the content of my sg_config.yml:
searchguard:
dynamic:
authc:
basic_internal_auth_domain:
enabled: true
order: 1
http_authenticator:
type: basic
challenge: true
authentication_backend:
type: intern
Any help will be greatly appreciated.