Problems with proxy users/roles

Search Guard
1

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

listen
${NGINX_PORT};

server_name
localhost;

location
${BASE_PATH} {

proxy_pass
http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

rewrite
/login http://localhost:4180/oauth2/sign_in redirect;

}

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:“email@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

xff:

enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

enabled: true

order: 0

http_authenticator:

type: proxy

challenge:
false

config:

user_header:
“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

type: noop

authc:

basic_internal_auth:

enabled: true

order: 1

http_authenticator:

type: basic

challenge:
true

authentication_backend:

type:
internal

sg_roles_mapping.yml

admin:

backendroles:

  • admin

host:

···

“*”

users:

  • elastic

  • user_name

kibana_user:

backendroles:

  • kibana_user

host:

“*”

users:

  • kibana

logstash_user:

backendroles:

logstash_user

host:

“*”

users:

  • logstash

beats_user:

backendroles:

  • beats_user

host:

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ

  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”

  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX

  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE

  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER

  • WRITE

  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

2

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

(“SSO configuration: Whitelisting HTTP headers”)

···

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"email@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

3

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

···

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

4

In the xff section of sg_config, can you try to add:

trustedProxies: ‘.*’ # trust all external proxies, regex pattern

And see if this solves the problem? Background:

searchguard.dynamic.http.xff. internalProxies:

“Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied.”

···

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

5

Its still exhibiting the same behavior.

···

On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:

In the xff section of sg_config, can you try to add:

trustedProxies: ‘.*’ # trust all external proxies, regex pattern

And see if this solves the problem? Background:

http://docs.search-guard.com/latest/search-guard-proxy

searchguard.dynamic.http.xff. internalProxies:

“Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied.”

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

6

Then I guess we need the Elasticsearch logs on debug level to see what’s going on behind the scenes. In order to enable it see here how to do it:

Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.

Then please make a request via your proxy, and attach the logfiles here.

···

On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:

Its still exhibiting the same behavior.

On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:

In the xff section of sg_config, can you try to add:

trustedProxies: ‘.*’ # trust all external proxies, regex pattern

And see if this solves the problem? Background:

http://docs.search-guard.com/latest/search-guard-proxy

searchguard.dynamic.http.xff. internalProxies:

“Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied.”

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

7

Please see the attached

elasticsearch-default.log (274 KB)

···

On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:

Then I guess we need the Elasticsearch logs on debug level to see what’s going on behind the scenes. In order to enable it see here how to do it:

http://docs.search-guard.com/latest/troubleshooting-tls#setting-the-log-level-to-debug

Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.

Then please make a request via your proxy, and attach the logfiles here.

On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:

Its still exhibiting the same behavior.

On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:

In the xff section of sg_config, can you try to add:

trustedProxies: ‘.*’ # trust all external proxies, regex pattern

And see if this solves the problem? Background:

http://docs.search-guard.com/latest/search-guard-proxy

searchguard.dynamic.http.xff. internalProxies:

“Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied.”

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

8

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator

org.elasticsearch.ElasticsearchSecurityException: xff not done

at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]

at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]

at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]

at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]

at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]

at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]

at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]

[2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator

[2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User ‘User [name=kibana, roles=[kibana_user], requestedTenant=null]’ is authenticated

[2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant ‘null’

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: ‘null’

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null

[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/: [indices:, indices:data/read*]

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for ‘kibana_user.?kibana’, evaluate other roles

[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>

···

On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:

Then I guess we need the Elasticsearch logs on debug level to see what’s going on behind the scenes. In order to enable it see here how to do it:

http://docs.search-guard.com/latest/troubleshooting-tls#setting-the-log-level-to-debug

Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.

Then please make a request via your proxy, and attach the logfiles here.

On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:

Its still exhibiting the same behavior.

On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:

In the xff section of sg_config, can you try to add:

trustedProxies: ‘.*’ # trust all external proxies, regex pattern

And see if this solves the problem? Background:

http://docs.search-guard.com/latest/search-guard-proxy

searchguard.dynamic.http.xff. internalProxies:

“Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied.”

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:

Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

When this setting…s value is true Kibana uses the

hostname specified in the server.host

setting. When the value of this setting is false, Kibana

uses the hostname of the host

that connects to this Kibana instance.

#elasticsearch.preserveHost: true

Kibana uses an index in Elasticsearch to store saved

searches, visualizations and

dashboards. Kibana creates a new index if the index

doesn…t already exist.

#kibana.index: “.kibana”

The default application to load.

#kibana.defaultAppId: “discover”

Kibana is served by a back end server. This setting

specifies the port to use.

server.port: 5601

Specifies the address to which the Kibana server will

bind. IP addresses and host names are both valid values.

The default is ‘localhost’, which usually means remote

machines will not be able to connect.

To allow connections from remote users, set this parameter

to a non-loopback address.

server.host: 0.0.0.0

Enables you to specify a path to mount Kibana at if you

are running behind a proxy. This only affects

the URLs generated by Kibana, your proxy is expected to

remove the basePath value before forwarding requests

to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

The maximum payload size in bytes for incoming server

requests.

#server.maxPayloadBytes: 1048576

The Kibana server’s name.

This is used for display purposes.

#server.name: “your-hostname”

The URL of the Elasticsearch instance to use for all your

queries.

elasticsearch.url: http://elasticsearch:9200

If your Elasticsearch is protected with basic

authentication, these settings provide

the username and password that the Kibana server uses to

perform maintenance on the Kibana

index at startup. Your Kibana users still need to

authenticate with Elasticsearch, which

is proxied through the Kibana server.

elasticsearch.username: “kibana”

elasticsearch.password: changeme

searchguard.cookie.password:
“4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6”

Paths to the PEM-format SSL certificate and SSL key files,

respectively. These

files enable SSL for outgoing requests from the Kibana

server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

Optional settings that provide the paths to the PEM-format

SSL certificate and key files.

These files validate that your Elasticsearch backend uses

the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

Optional setting that enables you to specify a path to the

PEM file for the certificate

authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities:
/etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

To disregard the validity of SSL certificates, change this

setting…s value to false.

elasticsearch.ssl.verificationMode: “certificate”

Time in milliseconds to wait for Elasticsearch to respond

to pings. Defaults to the value of

the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

Time in milliseconds to wait for responses from the back

end or Elasticsearch. This value

must be a positive integer.

#elasticsearch.requestTimeout: 30000

List of Kibana client-side headers to send to

Elasticsearch. To send no client-side

headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [
“authorization”, “x-forwarded-for”,
“x-forwarded-by”, “x-proxy-user”, “x-proxy-roles”
]

searchguard.basicauth.enabled: false

Header names and values that are sent to Elasticsearch.

Any custom headers cannot be overwritten

by client-side headers, regardless of the

elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

Time in milliseconds for Elasticsearch to wait for

responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

Time in milliseconds to wait for Elasticsearch at Kibana

startup before retrying.

#elasticsearch.startupTimeout: 5000

Specifies the path where Kibana creates the process ID

file.

#pid.file: /var/run/kibana.pid

Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

Set the value of this setting to true to suppress all

logging output.

#logging.silent: false

Set the value of this setting to true to suppress all

logging output other than error messages.

#logging.quiet: false

Set the value of this setting to true to log all events,

including system usage information

and all requests.

#logging.verbose: false

Set the interval in milliseconds to sample system and

process performance

metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:

Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

http://docs.search-guard.com/latest/kibana-authentication
(“SSO configuration: Whitelisting HTTP headers”)

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:

When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack
behind oauth2_proxy and nginx using search-guard as a authentication medium.
Once a user is authenticated they are forwarded to kibana correctly. However it
seems all users are assigned the internal kibana user and its associated role.
I would like to be able to assign a user as an admin role, which to my understanding
is done using the sg_roles_mapping.yml file, but it does not appear to be working
correctly. I can see the headers are passed to Kibana but elasticsearch does
not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

server {

    listen

${NGINX_PORT};

    server_name

localhost;

    location

${BASE_PATH} {

        proxy_pass

http://kibana:5601/;

proxy_set_header Authorization “Basic ${DIGEST}”;

proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

        rewrite

/login http://localhost:4180/oauth2/sign_in redirect;

    }

}

}

events {

worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 |
[2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level
perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null]
[IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]]
[RolesChecked [kibana_user]]

elasticsearch_1 |
[2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions
for {}

Kibana Log Entry

kibana_1 |
{“type”:“response”,“@timestamp”:“2018-01-30T15:54:47Z”,“tags”:,“pid”:28,“method”:“post”,“statusCode”:503,“req”:{“url”:“/api/saved_objects/index-pattern?overwrite=false”,“method”:“post”,“headers”:{“es-security-runas-user”:“user_name”,“x-proxy-user”:“user_name”,“x-proxy-roles”:“admin”,“host”:“kibana:5601”,“connection”:“close”,“content-length”:“66”,“user-agent”:“Mozilla/5.0
(Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko)
Chrome/63.0.3239.132
Safari/537.36”,“accept”:“application/json, text/plain,
/”,“accept-encoding”:“gzip, deflate,
br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/json;charset=UTF-8”,“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana”,“x-forwarded-email”:"em...@email.com",“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:“http://localhost:4180/kibana/app/kibana”},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:“POST
/api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B”}

sg_config.yml

searchguard:

dynamic:

http:

anonymous_auth_enabled: false

  xff:

    enabled: true

internalProxies: ‘.*’

remoteIpHeader: ‘x-forwarded-for’

proxiesHeader: ‘x-forwarded-by’

proxy_auth_domain:

  enabled: true

  order: 0

  http_authenticator:

    type: proxy

    challenge:

false

    config:

      user_header:

“x-proxy-user”

roles_header: “x-proxy-roles”

authentication_backend:

    type: noop

authc:

basic_internal_auth:

    enabled: true

    order: 1

http_authenticator:

      type: basic

      challenge:

true

authentication_backend:

      type:

internal

sg_roles_mapping.yml

admin:

backendroles:

    - admin

host:

    -

“*”

users:

    - elastic

    - user_name

kibana_user:

backendroles:

    - kibana_user

host:

    -

“*”

users:

    - kibana

logstash_user:

backendroles:

    -

logstash_user

host:

    -

“*”

users:

    - logstash

beats_user:

backendroles:

    - beats_user

host:

    -

“*”

users:

    - beats

sg_action_group.yml

INDICES

READ:

  • “indices:data/read*”

WRITE:

  • “indices:data/write*”

CRUD:

  • READ
  • WRITE

CREATE_INDEX:

  • “indices:admin/create*”
  • “indices:admin/mapping/put”

DELETE_INDEX:

  • “indices:admin/delete*”

INDEX_OWNER:

  • CREATE_INDEX
  • CRUD

INDEX_ALL:

  • “indices:*”

ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  • “indices:admin/template/get”

PUT_TEMPLATE:

  • “indices:admin/template/put”

TEMPLATE_OWNER:

  • GET_TEMPLATE
  • PUT_TEMPLATE

ES_INPUT:

  • TEMPLATE_OWNER
  • WRITE
  • MONITOR

MONITORING

MONITOR:

  • “cluster:monitor*”

SUPER POWERS

ALL:

  • “*”

Search Guard: “20.1”

ElasticSearch:6.1.1

9

"xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: '.*'

and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia

···

Am 31.01.2018 um 19:35 schrieb Kalada <kalada.opuiyo@t1cg.com>:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
  at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
  at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
  at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
  at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
  at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
  at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
  at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
  at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
  at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
  at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
  at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
  at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
  at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
[2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
[2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
[2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
[2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
[2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>

On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:

TLS help | Security for Elasticsearch | Search Guard

Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.

Then please make a request via your proxy, and attach the logfiles here.

On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
Its still exhibiting the same behavior.

On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
In the xff section of sg_config, can you try to add:

trustedProxies: '.*' # trust all external proxies, regex pattern

And see if this solves the problem? Background:

Search Guard Documentation 404 | Security for Elasticsearch | Search Guard

searchguard.dynamic.http.xff. internalProxies:

"Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."

On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below

# When this setting...s value is true Kibana uses the hostname specified in the server.host

# setting. When the value of this setting is false, Kibana uses the hostname of the host

# that connects to this Kibana instance.

#elasticsearch.preserveHost: true

# Kibana uses an index in Elasticsearch to store saved searches, visualizations and

# dashboards. Kibana creates a new index if the index doesn...t already exist.

#kibana.index: ".kibana"

# The default application to load.

#kibana.defaultAppId: "discover"

# Kibana is served by a back end server. This setting specifies the port to use.

server.port: 5601

# Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.

# The default is 'localhost', which usually means remote machines will not be able to connect.

# To allow connections from remote users, set this parameter to a non-loopback address.

server.host: 0.0.0.0

# Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects

# the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests

# to Kibana. This setting cannot end in a slash.

server.basePath: /kibana

# The maximum payload size in bytes for incoming server requests.

#server.maxPayloadBytes: 1048576

# The Kibana server's name. This is used for display purposes.

#server.name: "your-hostname"

# The URL of the Elasticsearch instance to use for all your queries.

elasticsearch.url: http://elasticsearch:9200

# If your Elasticsearch is protected with basic authentication, these settings provide

# the username and password that the Kibana server uses to perform maintenance on the Kibana

# index at startup. Your Kibana users still need to authenticate with Elasticsearch, which

# is proxied through the Kibana server.

elasticsearch.username: "kibana"

elasticsearch.password: changeme

searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"

# Paths to the PEM-format SSL certificate and SSL key files, respectively. These

# files enable SSL for outgoing requests from the Kibana server to the browser.

#server.ssl.cert: /path/to/your/server.crt

#server.ssl.key: /path/to/your/server.key

# Optional settings that provide the paths to the PEM-format SSL certificate and key files.

# These files validate that your Elasticsearch backend uses the same key files.

#elasticsearch.ssl.cert: /path/to/your/client.crt

#elasticsearch.ssl.key: /path/to/your/client.key

# Optional setting that enables you to specify a path to the PEM file for the certificate

# authority for your Elasticsearch instance.

elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem

# To disregard the validity of SSL certificates, change this setting...s value to false.

elasticsearch.ssl.verificationMode: "certificate"

# Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of

# the elasticsearch.requestTimeout setting.

#elasticsearch.pingTimeout: 1500

# Time in milliseconds to wait for responses from the back end or Elasticsearch. This value

# must be a positive integer.

#elasticsearch.requestTimeout: 30000

# List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side

# headers, set this value to (an empty list).

elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]

searchguard.basicauth.enabled: false

# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten

# by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.

#elasticsearch.customHeaders: {}

# Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.

#elasticsearch.shardTimeout: 0

# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.

#elasticsearch.startupTimeout: 5000

# Specifies the path where Kibana creates the process ID file.

#pid.file: /var/run/kibana.pid

# Enables you specify a file where Kibana stores log output.

#logging.dest: stdout

# Set the value of this setting to true to suppress all logging output.

#logging.silent: false

# Set the value of this setting to true to suppress all logging output other than error messages.

#logging.quiet: false

# Set the value of this setting to true to log all events, including system usage information

# and all requests.

#logging.verbose: false

# Set the interval in milliseconds to sample system and process performance

# metrics. Minimum is 100ms. Defaults to 5000.

#ops.interval: 5000

On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.

See also here for an example:

Kibana Authentication | Security for Elasticsearch | Search Guard
("SSO configuration: Whitelisting HTTP headers")

On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
When asking questions, please provide the following information:

Hi,

I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks

nginx.conf

http {

    server {

        listen ${NGINX_PORT};

        server_name localhost;

        location ${BASE_PATH} {

            proxy_pass http://kibana:5601/;

            proxy_set_header Authorization "Basic ${DIGEST}";

            proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;

            proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;

            proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;

            rewrite /login http://localhost:4180/oauth2/sign_in redirect;

        }

    }

}

events {

  worker_connections 1024;

}

Elastic search log entry

elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]

elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}

Kibana Log Entry

kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}

sg_config.yml

searchguard:

  dynamic:

    http:

      anonymous_auth_enabled: false

      xff:

        enabled: true

        internalProxies: '.*'

        remoteIpHeader: 'x-forwarded-for'

        proxiesHeader: 'x-forwarded-by'

    proxy_auth_domain:

      enabled: true

      order: 0

      http_authenticator:

        type: proxy

        challenge: false

        config:

          user_header: "x-proxy-user"

          roles_header: "x-proxy-roles"

      authentication_backend:

        type: noop

    authc:

      basic_internal_auth:

        enabled: true

        order: 1

        http_authenticator:

          type: basic

          challenge: true

        authentication_backend:

          type: internal

sg_roles_mapping.yml

admin:

    backendroles:

        - admin

    host:

        - "*"

    users:

        - elastic

        - user_name

kibana_user:

    backendroles:

        - kibana_user

    host:

        - "*"

    users:

        - kibana

logstash_user:

    backendroles:

        - logstash_user

    host:

        - "*"

    users:

        - logstash

beats_user:

    backendroles:

        - beats_user

    host:

        - "*"

    users:

        - beats

sg_action_group.yml

# INDICES

READ:

  - "indices:data/read*"

WRITE:

  - "indices:data/write*"

CRUD:

  - READ

  - WRITE

CREATE_INDEX:

  - "indices:admin/create*"

  - "indices:admin/mapping/put"

DELETE_INDEX:

  - "indices:admin/delete*"

INDEX_OWNER:

  - CREATE_INDEX

  - CRUD

INDEX_ALL:

  - "indices:*"

# ELASTICSEARCH ENTRYPOINT

GET_TEMPLATE:

  - "indices:admin/template/get"

PUT_TEMPLATE:

  - "indices:admin/template/put"

TEMPLATE_OWNER:

  - GET_TEMPLATE

  - PUT_TEMPLATE

ES_INPUT:

  - TEMPLATE_OWNER

  - WRITE

  - MONITOR

# MONITORING

MONITOR:

  - "cluster:monitor*"

# SUPER POWERS

ALL:

  - "*"

Search Guard: "20.1"

ElasticSearch:6.1.1

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.

10

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,

“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:

“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)

AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36",“accept”:"application/json, text/plain, /

",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,

“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,

“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},

“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},

“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

···

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:

“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator

org.elasticsearch.ElasticsearchSecurityException: xff not done

    at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

    at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

    at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

    at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

    at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]

    at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]

    at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]

    at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]

    at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]

    at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]

    at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

    at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
11

Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"

···

Am 31.01.2018 um 22:36 schrieb Kalada <kalada.opuiyo@t1cg.com>:

internalProxies: '.*' is the current setting. The headers appear to be valid please see an example below

kibana_1 | {"type":"response","@timestamp":"2018-01-31T20:14:37Z","tags":,"pid":21,"method":"post","statusCode":200,
"req":{"url":"/elasticsearch/_msearch","method":"post","headers":{"es-security-runas-user":"USERNAME","x-proxy-user":"USENAME","host":
"kibana:5601","connection":"close","content-length":"732","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36","accept":"application/json, text/plain, */*
","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/x-ndjson",
"kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:"http://localhost:4180/app/kibana&quot;,
"x-forwarded-email":"EMAIL","x-forwarded-for":"172.19.0.1","x-forwarded-user":"USERNAME"},
"remoteAddress":"172.19.0.3","userAgent":"172.19.0.3","referer":"http://localhost:4180/app/kibana&quot;\},
"res":{"statusCode":200,"responseTime":18,"contentLength":9},"message":"POST /elasticsearch/_msearch 200 18ms - 9.0B"}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
"xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: '.*'

and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia

> Am 31.01.2018 um 19:35 schrieb Kalada <kalada...@t1cg.com>:
>
> Hi Jochen,
>
> Playing with the setting I noticed this message now
>
> [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
> [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
> org.elasticsearch.ElasticsearchSecurityException: xff not done
> at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
> at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
> at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
> at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
> at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
> at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
> [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
> [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
> [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
> [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>
>
> On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
> Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:
>
> TLS help | Security for Elasticsearch | Search Guard
>
> Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.
>
> Then please make a request via your proxy, and attach the logfiles here.
>
> On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
> Its still exhibiting the same behavior.
>
> On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
> In the xff section of sg_config, can you try to add:
>
> trustedProxies: '.*' # trust all external proxies, regex pattern
>
> And see if this solves the problem? Background:
>
> Search Guard Documentation 404 | Security for Elasticsearch | Search Guard
>
> searchguard.dynamic.http.xff. internalProxies:
>
> "Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."
>
> On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
> Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below
>
> # When this setting...s value is true Kibana uses the hostname specified in the server.host
>
> # setting. When the value of this setting is false, Kibana uses the hostname of the host
>
> # that connects to this Kibana instance.
>
> #elasticsearch.preserveHost: true
>
>
> # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
>
> # dashboards. Kibana creates a new index if the index doesn...t already exist.
>
> #kibana.index: ".kibana"
>
>
> # The default application to load.
>
> #kibana.defaultAppId: "discover"
>
>
> # Kibana is served by a back end server. This setting specifies the port to use.
>
> server.port: 5601
>
>
> # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
>
> # The default is 'localhost', which usually means remote machines will not be able to connect.
>
> # To allow connections from remote users, set this parameter to a non-loopback address.
>
> server.host: 0.0.0.0
>
>
> # Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
>
> # the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
>
> # to Kibana. This setting cannot end in a slash.
>
> server.basePath: /kibana
>
>
> # The maximum payload size in bytes for incoming server requests.
>
> #server.maxPayloadBytes: 1048576
>
>
> # The Kibana server's name. This is used for display purposes.
>
> #server.name: "your-hostname"
>
>
> # The URL of the Elasticsearch instance to use for all your queries.
>
> elasticsearch.url: http://elasticsearch:9200
>
> # If your Elasticsearch is protected with basic authentication, these settings provide
>
> # the username and password that the Kibana server uses to perform maintenance on the Kibana
>
> # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
>
> # is proxied through the Kibana server.
>
> elasticsearch.username: "kibana"
>
> elasticsearch.password: changeme
>
> searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"
>
>
> # Paths to the PEM-format SSL certificate and SSL key files, respectively. These
>
> # files enable SSL for outgoing requests from the Kibana server to the browser.
>
> #server.ssl.cert: /path/to/your/server.crt
>
> #server.ssl.key: /path/to/your/server.key
>
>
> # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
>
> # These files validate that your Elasticsearch backend uses the same key files.
>
> #elasticsearch.ssl.cert: /path/to/your/client.crt
>
> #elasticsearch.ssl.key: /path/to/your/client.key
>
>
> # Optional setting that enables you to specify a path to the PEM file for the certificate
>
> # authority for your Elasticsearch instance.
>
> elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem
>
>
> # To disregard the validity of SSL certificates, change this setting...s value to false.
>
> elasticsearch.ssl.verificationMode: "certificate"
>
> # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
>
> # the elasticsearch.requestTimeout setting.
>
> #elasticsearch.pingTimeout: 1500
>
>
> # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
>
> # must be a positive integer.
>
> #elasticsearch.requestTimeout: 30000
>
>
> # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
>
> # headers, set this value to (an empty list).
>
> elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]
>
> searchguard.basicauth.enabled: false
>
>
> # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
>
> # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
>
> #elasticsearch.customHeaders: {}
>
>
> # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
>
> #elasticsearch.shardTimeout: 0
>
>
> # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
>
> #elasticsearch.startupTimeout: 5000
>
>
> # Specifies the path where Kibana creates the process ID file.
>
> #pid.file: /var/run/kibana.pid
>
>
> # Enables you specify a file where Kibana stores log output.
>
> #logging.dest: stdout
>
>
> # Set the value of this setting to true to suppress all logging output.
>
> #logging.silent: false
>
>
> # Set the value of this setting to true to suppress all logging output other than error messages.
>
> #logging.quiet: false
>
>
> # Set the value of this setting to true to log all events, including system usage information
>
> # and all requests.
>
> #logging.verbose: false
>
>
> # Set the interval in milliseconds to sample system and process performance
>
> # metrics. Minimum is 100ms. Defaults to 5000.
>
> #ops.interval: 5000
>
>
> On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
> Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.
>
> See also here for an example:
>
> Kibana Authentication | Security for Elasticsearch | Search Guard
> ("SSO configuration: Whitelisting HTTP headers")
>
> On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
> When asking questions, please provide the following information:
>
> Hi,
>
>
> I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks
>
>
>
> nginx.conf
>
> http {
>
> server {
>
> listen ${NGINX_PORT};
>
> server_name localhost;
>
>
> location ${BASE_PATH} {
>
>
> proxy_pass http://kibana:5601/;
>
>
> proxy_set_header Authorization "Basic ${DIGEST}";
>
>
> proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;
>
> proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;
>
> proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;
>
>
> rewrite /login http://localhost:4180/oauth2/sign_in redirect;
>
> }
>
> }
>
>
>
> }
>
>
> events {
>
> worker_connections 1024;
>
> }
>
>
> Elastic search log entry
>
>
> elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]
>
> elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}
>
>
> Kibana Log Entry
>
> kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}
>
>
> sg_config.yml
>
>
> searchguard:
>
> dynamic:
>
> http:
>
> anonymous_auth_enabled: false
>
> xff:
>
> enabled: true
>
> internalProxies: '.*'
>
> remoteIpHeader: 'x-forwarded-for'
>
> proxiesHeader: 'x-forwarded-by'
>
> proxy_auth_domain:
>
> enabled: true
>
> order: 0
>
> http_authenticator:
>
> type: proxy
>
> challenge: false
>
> config:
>
> user_header: "x-proxy-user"
>
> roles_header: "x-proxy-roles"
>
> authentication_backend:
>
> type: noop
>
> authc:
>
> basic_internal_auth:
>
> enabled: true
>
> order: 1
>
> http_authenticator:
>
> type: basic
>
> challenge: true
>
> authentication_backend:
>
> type: internal
>
>
>
> sg_roles_mapping.yml
>
>
> admin:
>
> backendroles:
>
> - admin
>
> host:
>
> - "*"
>
> users:
>
> - elastic
>
> - user_name
>
> kibana_user:
>
> backendroles:
>
> - kibana_user
>
> host:
>
> - "*"
>
> users:
>
> - kibana
>
>
> logstash_user:
>
> backendroles:
>
> - logstash_user
>
> host:
>
> - "*"
>
> users:
>
> - logstash
>
>
> beats_user:
>
> backendroles:
>
> - beats_user
>
> host:
>
> - "*"
>
> users:
>
> - beats
>
>
>
> sg_action_group.yml
>
>
>
> # INDICES
>
> READ:
>
> - "indices:data/read*"
>
> WRITE:
>
> - "indices:data/write*"
>
> CRUD:
>
> - READ
>
> - WRITE
>
> CREATE_INDEX:
>
> - "indices:admin/create*"
>
> - "indices:admin/mapping/put"
>
> DELETE_INDEX:
>
> - "indices:admin/delete*"
>
> INDEX_OWNER:
>
> - CREATE_INDEX
>
> - CRUD
>
> INDEX_ALL:
>
> - "indices:*"
>
>
>
> # ELASTICSEARCH ENTRYPOINT
>
> GET_TEMPLATE:
>
> - "indices:admin/template/get"
>
> PUT_TEMPLATE:
>
> - "indices:admin/template/put"
>
> TEMPLATE_OWNER:
>
> - GET_TEMPLATE
>
> - PUT_TEMPLATE
>
> ES_INPUT:
>
> - TEMPLATE_OWNER
>
> - WRITE
>
> - MONITOR
>
>
>
> # MONITORING
>
> MONITOR:
>
> - "cluster:monitor*"
>
>
>
> # SUPER POWERS
>
> ALL:
>
>
> - "*"
>
>
>
> Search Guard: "20.1"
>
> ElasticSearch:6.1.1
>
>
>
>
>
>
> --
> You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> To post to this group, send email to search...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
> For more options, visit https://groups.google.com/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/3ba4a31b-2667-4b82-89e7-c95fcdb98caf%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.

12

This is the message I received

  • Rebuilt URL to: https://elasticsearch:9200/

  • Trying 172.22.0.3…

  • TCP_NODELAY set

  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)

  • ALPN, offering http/1.1

  • successfully set certificate verify locations:

  • CAfile: /etc/ssl/certs/ca-certificates.crt

CApath: none

  • TLSv1.2 (OUT), TLS handshake, Client hello (1):

  • error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

  • Closing connection 0

curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

···

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:

Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

Am 31.01.2018 um 22:36 schrieb Kalada kalada...@t1cg.com:

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,

“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:

“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36”,“accept”:"application/json, text/plain, /

",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,

“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,

“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},

“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},

“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:

“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.

13

Have you https enabled? If not its
curl -sS http://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
curl --version

···

Am 01.02.2018 um 22:27 schrieb Kalada <kalada.opuiyo@t1cg.com>:

This is the messages I received

* Rebuilt URL to: https://elasticsearch:9200/
* Trying 172.22.0.3...
* TCP_NODELAY set
* Connected to elasticsearch (172.22.0.3) port 9200 (#0)
* ALPN, offering http/1.1
* successfully set certificate verify locations:
* CAfile: /etc/ssl/certs/ca-certificates.crt
  CApath: none
* TLSv1.2 (OUT), TLS handshake, Client hello (1):
* error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
* Closing connection 0
curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"

> Am 31.01.2018 um 22:36 schrieb Kalada <kalada...@t1cg.com>:
>
> internalProxies: '.*' is the current setting. The headers appear to be valid please see an example below
>
> kibana_1 | {"type":"response","@timestamp":"2018-01-31T20:14:37Z","tags":,"pid":21,"method":"post","statusCode":200,
> "req":{"url":"/elasticsearch/_msearch","method":"post","headers":{"es-security-runas-user":"USERNAME","x-proxy-user":"USENAME","host":
> "kibana:5601","connection":"close","content-length":"732","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
> AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36","accept":"application/json, text/plain, */*
> ","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/x-ndjson",
> "kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/app/kibana&quot;,
> "x-forwarded-email":"EMAIL","x-forwarded-for":"172.19.0.1","x-forwarded-user":"USERNAME"},
> "remoteAddress":"172.19.0.3","userAgent":"172.19.0.3","referer":"http://localhost:4180/app/kibana&quot;\},
> "res":{"statusCode":200,"responseTime":18,"contentLength":9},"message":"POST /elasticsearch/_msearch 200 18ms - 9.0B"}
>
>
>
> On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
> "xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present
>
> you can test ist by allowing simply all internal proxies
>
> internalProxies: '.*'
>
> and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia
>
> > Am 31.01.2018 um 19:35 schrieb Kalada <kalada...@t1cg.com>:
> >
> > Hi Jochen,
> >
> > Playing with the setting I noticed this message now
> >
> > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
> > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
> > org.elasticsearch.ElasticsearchSecurityException: xff not done
> > at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
> > at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
> > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
> > at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
> > at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
> > at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
> > at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
> > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
> > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
> > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
> > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>
> >
> > On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
> > Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:
> >
> > TLS help | Security for Elasticsearch | Search Guard
> >
> > Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.
> >
> > Then please make a request via your proxy, and attach the logfiles here.
> >
> > On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
> > Its still exhibiting the same behavior.
> >
> > On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
> > In the xff section of sg_config, can you try to add:
> >
> > trustedProxies: '.*' # trust all external proxies, regex pattern
> >
> > And see if this solves the problem? Background:
> >
> > Search Guard Documentation 404 | Security for Elasticsearch | Search Guard
> >
> > searchguard.dynamic.http.xff. internalProxies:
> >
> > "Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."
> >
> > On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
> > Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below
> >
> > # When this setting...s value is true Kibana uses the hostname specified in the server.host
> >
> > # setting. When the value of this setting is false, Kibana uses the hostname of the host
> >
> > # that connects to this Kibana instance.
> >
> > #elasticsearch.preserveHost: true
> >
> >
> > # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
> >
> > # dashboards. Kibana creates a new index if the index doesn...t already exist.
> >
> > #kibana.index: ".kibana"
> >
> >
> > # The default application to load.
> >
> > #kibana.defaultAppId: "discover"
> >
> >
> > # Kibana is served by a back end server. This setting specifies the port to use.
> >
> > server.port: 5601
> >
> >
> > # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
> >
> > # The default is 'localhost', which usually means remote machines will not be able to connect.
> >
> > # To allow connections from remote users, set this parameter to a non-loopback address.
> >
> > server.host: 0.0.0.0
> >
> >
> > # Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
> >
> > # the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
> >
> > # to Kibana. This setting cannot end in a slash.
> >
> > server.basePath: /kibana
> >
> >
> > # The maximum payload size in bytes for incoming server requests.
> >
> > #server.maxPayloadBytes: 1048576
> >
> >
> > # The Kibana server's name. This is used for display purposes.
> >
> > #server.name: "your-hostname"
> >
> >
> > # The URL of the Elasticsearch instance to use for all your queries.
> >
> > elasticsearch.url: http://elasticsearch:9200
> >
> > # If your Elasticsearch is protected with basic authentication, these settings provide
> >
> > # the username and password that the Kibana server uses to perform maintenance on the Kibana
> >
> > # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
> >
> > # is proxied through the Kibana server.
> >
> > elasticsearch.username: "kibana"
> >
> > elasticsearch.password: changeme
> >
> > searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"
> >
> >
> > # Paths to the PEM-format SSL certificate and SSL key files, respectively. These
> >
> > # files enable SSL for outgoing requests from the Kibana server to the browser.
> >
> > #server.ssl.cert: /path/to/your/server.crt
> >
> > #server.ssl.key: /path/to/your/server.key
> >
> >
> > # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
> >
> > # These files validate that your Elasticsearch backend uses the same key files.
> >
> > #elasticsearch.ssl.cert: /path/to/your/client.crt
> >
> > #elasticsearch.ssl.key: /path/to/your/client.key
> >
> >
> > # Optional setting that enables you to specify a path to the PEM file for the certificate
> >
> > # authority for your Elasticsearch instance.
> >
> > elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem
> >
> >
> > # To disregard the validity of SSL certificates, change this setting...s value to false.
> >
> > elasticsearch.ssl.verificationMode: "certificate"
> >
> > # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
> >
> > # the elasticsearch.requestTimeout setting.
> >
> > #elasticsearch.pingTimeout: 1500
> >
> >
> > # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
> >
> > # must be a positive integer.
> >
> > #elasticsearch.requestTimeout: 30000
> >
> >
> > # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
> >
> > # headers, set this value to (an empty list).
> >
> > elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]
> >
> > searchguard.basicauth.enabled: false
> >
> >
> > # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
> >
> > # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
> >
> > #elasticsearch.customHeaders: {}
> >
> >
> > # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
> >
> > #elasticsearch.shardTimeout: 0
> >
> >
> > # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
> >
> > #elasticsearch.startupTimeout: 5000
> >
> >
> > # Specifies the path where Kibana creates the process ID file.
> >
> > #pid.file: /var/run/kibana.pid
> >
> >
> > # Enables you specify a file where Kibana stores log output.
> >
> > #logging.dest: stdout
> >
> >
> > # Set the value of this setting to true to suppress all logging output.
> >
> > #logging.silent: false
> >
> >
> > # Set the value of this setting to true to suppress all logging output other than error messages.
> >
> > #logging.quiet: false
> >
> >
> > # Set the value of this setting to true to log all events, including system usage information
> >
> > # and all requests.
> >
> > #logging.verbose: false
> >
> >
> > # Set the interval in milliseconds to sample system and process performance
> >
> > # metrics. Minimum is 100ms. Defaults to 5000.
> >
> > #ops.interval: 5000
> >
> >
> > On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
> > Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.
> >
> > See also here for an example:
> >
> > Kibana Authentication | Security for Elasticsearch | Search Guard
> > ("SSO configuration: Whitelisting HTTP headers")
> >
> > On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
> > When asking questions, please provide the following information:
> >
> > Hi,
> >
> >
> > I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks
> >
> >
> >
> > nginx.conf
> >
> > http {
> >
> > server {
> >
> > listen ${NGINX_PORT};
> >
> > server_name localhost;
> >
> >
> > location ${BASE_PATH} {
> >
> >
> > proxy_pass http://kibana:5601/;
> >
> >
> > proxy_set_header Authorization "Basic ${DIGEST}";
> >
> >
> > proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;
> >
> > proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;
> >
> > proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;
> >
> >
> > rewrite /login http://localhost:4180/oauth2/sign_in redirect;
> >
> > }
> >
> > }
> >
> >
> >
> > }
> >
> >
> > events {
> >
> > worker_connections 1024;
> >
> > }
> >
> >
> > Elastic search log entry
> >
> >
> > elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]
> >
> > elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}
> >
> >
> > Kibana Log Entry
> >
> > kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}
> >
> >
> > sg_config.yml
> >
> >
> > searchguard:
> >
> > dynamic:
> >
> > http:
> >
> > anonymous_auth_enabled: false
> >
> > xff:
> >
> > enabled: true
> >
> > internalProxies: '.*'
> >
> > remoteIpHeader: 'x-forwarded-for'
> >
> > proxiesHeader: 'x-forwarded-by'
> >
> > proxy_auth_domain:
> >
> > enabled: true
> >
> > order: 0
> >
> > http_authenticator:
> >
> > type: proxy
> >
> > challenge: false
> >
> > config:
> >
> > user_header: "x-proxy-user"
> >
> > roles_header: "x-proxy-roles"
> >
> > authentication_backend:
> >
> > type: noop
> >
> > authc:
> >
> > basic_internal_auth:
> >
> > enabled: true
> >
> > order: 1
> >
> > http_authenticator:
> >
> > type: basic
> >
> > challenge: true
> >
> > authentication_backend:
> >
> > type: internal
> >
> >
> >
> > sg_roles_mapping.yml
> >
> >
> > admin:
> >
> > backendroles:
> >
> > - admin
> >
> > host:
> >
> > - "*"
> >
> > users:
> >
> > - elastic
> >
> > - user_name
> >
> > kibana_user:
> >
> > backendroles:
> >
> > - kibana_user
> >
> > host:
> >
> > - "*"
> >
> > users:
> >
> > - kibana
> >
> >
> > logstash_user:
> >
> > backendroles:
> >
> > - logstash_user
> >
> > host:
> >
> > - "*"
> >
> > users:
> >
> > - logstash
> >
> >
> > beats_user:
> >
> > backendroles:
> >
> > - beats_user
> >
> > host:
> >
> > - "*"
> >
> > users:
> >
> > - beats
> >
> >
> >
> > sg_action_group.yml
> >
> >
> >
> > # INDICES
> >
> > READ:
> >
> > - "indices:data/read*"
> >
> > WRITE:
> >
> > - "indices:data/write*"
> >
> > CRUD:
> >
> > - READ
> >
> > - WRITE
> >
> > CREATE_INDEX:
> >
> > - "indices:admin/create*"
> >
> > - "indices:admin/mapping/put"
> >
> > DELETE_INDEX:
> >
> > - "indices:admin/delete*"
> >
> > INDEX_OWNER:
> >
> > - CREATE_INDEX
> >
> > - CRUD
> >
> > INDEX_ALL:
> >
> > - "indices:*"
> >
> >
> >
> > # ELASTICSEARCH ENTRYPOINT
> >
> > GET_TEMPLATE:
> >
> > - "indices:admin/template/get"
> >
> > PUT_TEMPLATE:
> >
> > - "indices:admin/template/put"
> >
> > TEMPLATE_OWNER:
> >
> > - GET_TEMPLATE
> >
> > - PUT_TEMPLATE
> >
> > ES_INPUT:
> >
> > - TEMPLATE_OWNER
> >
> > - WRITE
> >
> > - MONITOR
> >
> >
> >
> > # MONITORING
> >
> > MONITOR:
> >
> > - "cluster:monitor*"
> >
> >
> >
> > # SUPER POWERS
> >
> > ALL:
> >
> >
> > - "*"
> >
> >
> >
> > Search Guard: "20.1"
> >
> > ElasticSearch:6.1.1
> >
> >
> >
> >
> >
> >
> > --
> > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > To post to this group, send email to search...@googlegroups.com.
> > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
> > For more options, visit https://groups.google.com/d/optout\.
>
>
> --
> You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> To post to this group, send email to search...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/3ba4a31b-2667-4b82-89e7-c95fcdb98caf%40googlegroups.com\.
> For more options, visit https://groups.google.com/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/554d479e-fdd9-4357-ba3d-32a2d03d2699%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.

14

Thank you https wasnt enabled please see below

  • Rebuilt URL to: http://elasticsearch:9200/

  • Trying 172.22.0.3…

  • TCP_NODELAY set

  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)

GET / HTTP/1.1

Host: elasticsearch:9200

User-Agent: curl/7.57.0

Accept: /

x-proxy-user: myuser

x-proxy-roles: myrole

x-forwarded-for: 127.0.0.1

< HTTP/1.1 401 Unauthorized

< WWW-Authenticate: Basic realm=“Search Guard”

< content-type: text/plain; charset=UTF-8

< content-length: 12

<

  • Connection #0 to host elasticsearch left intact
···

On Thursday, February 1, 2018 at 4:41:37 PM UTC-5, Search Guard wrote:

Have you https enabled? If not its

curl -sS http://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.

curl --version

Am 01.02.2018 um 22:27 schrieb Kalada kalada...@t1cg.com:

This is the messages I received

  • Trying 172.22.0.3…
  • TCP_NODELAY set
  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)
  • ALPN, offering http/1.1
  • successfully set certificate verify locations:
  • CAfile: /etc/ssl/certs/ca-certificates.crt

CApath: none

  • TLSv1.2 (OUT), TLS handshake, Client hello (1):
  • error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
  • Closing connection 0

curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:

Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

Am 31.01.2018 um 22:36 schrieb Kalada kalada...@t1cg.com:

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,
“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:
“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36”,“accept”:"application/json, text/plain, /
",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,
“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,
“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},
“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},
“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]

15

pls provide your sg_config.yml, elasticsearch.yml and the logs (as files/attachments)

···

Am 01.02.2018 um 23:04 schrieb Kalada <kalada.opuiyo@t1cg.com>:

Thank you https wasnt enabled please see below

* Rebuilt URL to: http://elasticsearch:9200/
* Trying 172.22.0.3...
* TCP_NODELAY set
* Connected to elasticsearch (172.22.0.3) port 9200 (#0)
> GET / HTTP/1.1
> Host: elasticsearch:9200
> User-Agent: curl/7.57.0
> Accept: */*
> x-proxy-user: myuser
> x-proxy-roles: myrole
> x-forwarded-for: 127.0.0.1
>
< HTTP/1.1 401 Unauthorized
< WWW-Authenticate: Basic realm="Search Guard"
< content-type: text/plain; charset=UTF-8
< content-length: 12
<
* Connection #0 to host elasticsearch left intact

On Thursday, February 1, 2018 at 4:41:37 PM UTC-5, Search Guard wrote:
Have you https enabled? If not its
curl -sS http://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
curl --version

> Am 01.02.2018 um 22:27 schrieb Kalada <kalada...@t1cg.com>:
>
> This is the messages I received
>
> * Rebuilt URL to: https://elasticsearch:9200/
> * Trying 172.22.0.3...
> * TCP_NODELAY set
> * Connected to elasticsearch (172.22.0.3) port 9200 (#0)
> * ALPN, offering http/1.1
> * successfully set certificate verify locations:
> * CAfile: /etc/ssl/certs/ca-certificates.crt
> CApath: none
> * TLSv1.2 (OUT), TLS handshake, Client hello (1):
> * error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
> * Closing connection 0
> curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
>
> On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
> Can you test it via curl or postman (without kibana) like
>
> curl -sSk https://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"
>
> > Am 31.01.2018 um 22:36 schrieb Kalada <kalada...@t1cg.com>:
> >
> > internalProxies: '.*' is the current setting. The headers appear to be valid please see an example below
> >
> > kibana_1 | {"type":"response","@timestamp":"2018-01-31T20:14:37Z","tags":,"pid":21,"method":"post","statusCode":200,
> > "req":{"url":"/elasticsearch/_msearch","method":"post","headers":{"es-security-runas-user":"USERNAME","x-proxy-user":"USENAME","host":
> > "kibana:5601","connection":"close","content-length":"732","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
> > AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36","accept":"application/json, text/plain, */*
> > ","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/x-ndjson",
> > "kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/app/kibana&quot;,
> > "x-forwarded-email":"EMAIL","x-forwarded-for":"172.19.0.1","x-forwarded-user":"USERNAME"},
> > "remoteAddress":"172.19.0.3","userAgent":"172.19.0.3","referer":"http://localhost:4180/app/kibana&quot;\},
> > "res":{"statusCode":200,"responseTime":18,"contentLength":9},"message":"POST /elasticsearch/_msearch 200 18ms - 9.0B"}
> >
> >
> >
> > On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
> > "xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present
> >
> > you can test ist by allowing simply all internal proxies
> >
> > internalProxies: '.*'
> >
> > and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia
> >
> > > Am 31.01.2018 um 19:35 schrieb Kalada <kalada...@t1cg.com>:
> > >
> > > Hi Jochen,
> > >
> > > Playing with the setting I noticed this message now
> > >
> > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
> > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
> > > org.elasticsearch.ElasticsearchSecurityException: xff not done
> > > at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
> > > at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
> > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
> > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>
> > >
> > > On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
> > > Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:
> > >
> > > TLS help | Security for Elasticsearch | Search Guard
> > >
> > > Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.
> > >
> > > Then please make a request via your proxy, and attach the logfiles here.
> > >
> > > On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
> > > Its still exhibiting the same behavior.
> > >
> > > On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
> > > In the xff section of sg_config, can you try to add:
> > >
> > > trustedProxies: '.*' # trust all external proxies, regex pattern
> > >
> > > And see if this solves the problem? Background:
> > >
> > > Search Guard Documentation 404 | Security for Elasticsearch | Search Guard
> > >
> > > searchguard.dynamic.http.xff. internalProxies:
> > >
> > > "Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."
> > >
> > > On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
> > > Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below
> > >
> > > # When this setting...s value is true Kibana uses the hostname specified in the server.host
> > >
> > > # setting. When the value of this setting is false, Kibana uses the hostname of the host
> > >
> > > # that connects to this Kibana instance.
> > >
> > > #elasticsearch.preserveHost: true
> > >
> > >
> > > # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
> > >
> > > # dashboards. Kibana creates a new index if the index doesn...t already exist.
> > >
> > > #kibana.index: ".kibana"
> > >
> > >
> > > # The default application to load.
> > >
> > > #kibana.defaultAppId: "discover"
> > >
> > >
> > > # Kibana is served by a back end server. This setting specifies the port to use.
> > >
> > > server.port: 5601
> > >
> > >
> > > # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
> > >
> > > # The default is 'localhost', which usually means remote machines will not be able to connect.
> > >
> > > # To allow connections from remote users, set this parameter to a non-loopback address.
> > >
> > > server.host: 0.0.0.0
> > >
> > >
> > > # Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
> > >
> > > # the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
> > >
> > > # to Kibana. This setting cannot end in a slash.
> > >
> > > server.basePath: /kibana
> > >
> > >
> > > # The maximum payload size in bytes for incoming server requests.
> > >
> > > #server.maxPayloadBytes: 1048576
> > >
> > >
> > > # The Kibana server's name. This is used for display purposes.
> > >
> > > #server.name: "your-hostname"
> > >
> > >
> > > # The URL of the Elasticsearch instance to use for all your queries.
> > >
> > > elasticsearch.url: http://elasticsearch:9200
> > >
> > > # If your Elasticsearch is protected with basic authentication, these settings provide
> > >
> > > # the username and password that the Kibana server uses to perform maintenance on the Kibana
> > >
> > > # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
> > >
> > > # is proxied through the Kibana server.
> > >
> > > elasticsearch.username: "kibana"
> > >
> > > elasticsearch.password: changeme
> > >
> > > searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"
> > >
> > >
> > > # Paths to the PEM-format SSL certificate and SSL key files, respectively. These
> > >
> > > # files enable SSL for outgoing requests from the Kibana server to the browser.
> > >
> > > #server.ssl.cert: /path/to/your/server.crt
> > >
> > > #server.ssl.key: /path/to/your/server.key
> > >
> > >
> > > # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
> > >
> > > # These files validate that your Elasticsearch backend uses the same key files.
> > >
> > > #elasticsearch.ssl.cert: /path/to/your/client.crt
> > >
> > > #elasticsearch.ssl.key: /path/to/your/client.key
> > >
> > >
> > > # Optional setting that enables you to specify a path to the PEM file for the certificate
> > >
> > > # authority for your Elasticsearch instance.
> > >
> > > elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem
> > >
> > >
> > > # To disregard the validity of SSL certificates, change this setting...s value to false.
> > >
> > > elasticsearch.ssl.verificationMode: "certificate"
> > >
> > > # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
> > >
> > > # the elasticsearch.requestTimeout setting.
> > >
> > > #elasticsearch.pingTimeout: 1500
> > >
> > >
> > > # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
> > >
> > > # must be a positive integer.
> > >
> > > #elasticsearch.requestTimeout: 30000
> > >
> > >
> > > # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
> > >
> > > # headers, set this value to (an empty list).
> > >
> > > elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]
> > >
> > > searchguard.basicauth.enabled: false
> > >
> > >
> > > # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
> > >
> > > # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
> > >
> > > #elasticsearch.customHeaders: {}
> > >
> > >
> > > # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
> > >
> > > #elasticsearch.shardTimeout: 0
> > >
> > >
> > > # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
> > >
> > > #elasticsearch.startupTimeout: 5000
> > >
> > >
> > > # Specifies the path where Kibana creates the process ID file.
> > >
> > > #pid.file: /var/run/kibana.pid
> > >
> > >
> > > # Enables you specify a file where Kibana stores log output.
> > >
> > > #logging.dest: stdout
> > >
> > >
> > > # Set the value of this setting to true to suppress all logging output.
> > >
> > > #logging.silent: false
> > >
> > >
> > > # Set the value of this setting to true to suppress all logging output other than error messages.
> > >
> > > #logging.quiet: false
> > >
> > >
> > > # Set the value of this setting to true to log all events, including system usage information
> > >
> > > # and all requests.
> > >
> > > #logging.verbose: false
> > >
> > >
> > > # Set the interval in milliseconds to sample system and process performance
> > >
> > > # metrics. Minimum is 100ms. Defaults to 5000.
> > >
> > > #ops.interval: 5000
> > >
> > >
> > > On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
> > > Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.
> > >
> > > See also here for an example:
> > >
> > > Kibana Authentication | Security for Elasticsearch | Search Guard
> > > ("SSO configuration: Whitelisting HTTP headers")
> > >
> > > On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
> > > When asking questions, please provide the following information:
> > >
> > > Hi,
> > >
> > >
> > > I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks
> > >
> > >
> > >
> > > nginx.conf
> > >
> > > http {
> > >
> > > server {
> > >
> > > listen ${NGINX_PORT};
> > >
> > > server_name localhost;
> > >
> > >
> > > location ${BASE_PATH} {
> > >
> > >
> > > proxy_pass http://kibana:5601/;
> > >
> > >
> > > proxy_set_header Authorization "Basic ${DIGEST}";
> > >
> > >
> > > proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;
> > >
> > > proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;
> > >
> > > proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;
> > >
> > >
> > > rewrite /login http://localhost:4180/oauth2/sign_in redirect;
> > >
> > > }
> > >
> > > }
> > >
> > >
> > >
> > > }
> > >
> > >
> > > events {
> > >
> > > worker_connections 1024;
> > >
> > > }
> > >
> > >
> > > Elastic search log entry
> > >
> > >
> > > elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]
> > >
> > > elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}
> > >
> > >
> > > Kibana Log Entry
> > >
> > > kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}
> > >
> > >
> > > sg_config.yml
> > >
> > >
> > > searchguard:
> > >
> > > dynamic:
> > >
> > > http:
> > >
> > > anonymous_auth_enabled: false
> > >
> > > xff:
> > >
> > > enabled: true
> > >
> > > internalProxies: '.*'
> > >
> > > remoteIpHeader: 'x-forwarded-for'
> > >
> > > proxiesHeader: 'x-forwarded-by'
> > >
> > > proxy_auth_domain:
> > >
> > > enabled: true
> > >
> > > order: 0
> > >
> > > http_authenticator:
> > >
> > > type: proxy
> > >
> > > challenge: false
> > >
> > > config:
> > >
> > > user_header: "x-proxy-user"
> > >
> > > roles_header: "x-proxy-roles"
> > >
> > > authentication_backend:
> > >
> > > type: noop
> > >
> > > authc:
> > >
> > > basic_internal_auth:
> > >
> > > enabled: true
> > >
> > > order: 1
> > >
> > > http_authenticator:
> > >
> > > type: basic
> > >
> > > challenge: true
> > >
> > > authentication_backend:
> > >
> > > type: internal
> > >
> > >
> > >
> > > sg_roles_mapping.yml
> > >
> > >
> > > admin:
> > >
> > > backendroles:
> > >
> > > - admin
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - elastic
> > >
> > > - user_name
> > >
> > > kibana_user:
> > >
> > > backendroles:
> > >
> > > - kibana_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - kibana
> > >
> > >
> > > logstash_user:
> > >
> > > backendroles:
> > >
> > > - logstash_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - logstash
> > >
> > >
> > > beats_user:
> > >
> > > backendroles:
> > >
> > > - beats_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - beats
> > >
> > >
> > >
> > > sg_action_group.yml
> > >
> > >
> > >
> > > # INDICES
> > >
> > > READ:
> > >
> > > - "indices:data/read*"
> > >
> > > WRITE:
> > >
> > > - "indices:data/write*"
> > >
> > > CRUD:
> > >
> > > - READ
> > >
> > > - WRITE
> > >
> > > CREATE_INDEX:
> > >
> > > - "indices:admin/create*"
> > >
> > > - "indices:admin/mapping/put"
> > >
> > > DELETE_INDEX:
> > >
> > > - "indices:admin/delete*"
> > >
> > > INDEX_OWNER:
> > >
> > > - CREATE_INDEX
> > >
> > > - CRUD
> > >
> > > INDEX_ALL:
> > >
> > > - "indices:*"
> > >
> > >
> > >
> > > # ELASTICSEARCH ENTRYPOINT
> > >
> > > GET_TEMPLATE:
> > >
> > > - "indices:admin/template/get"
> > >
> > > PUT_TEMPLATE:
> > >
> > > - "indices:admin/template/put"
> > >
> > > TEMPLATE_OWNER:
> > >
> > > - GET_TEMPLATE
> > >
> > > - PUT_TEMPLATE
> > >
> > > ES_INPUT:
> > >
> > > - TEMPLATE_OWNER
> > >
> > > - WRITE
> > >
> > > - MONITOR
> > >
> > >
> > >
> > > # MONITORING
> > >
> > > MONITOR:
> > >
> > > - "cluster:monitor*"
> > >
> > >
> > >
> > > # SUPER POWERS
> > >
> > > ALL:
> > >
> > >
> > > - "*"
> > >
> > >
> > >
> > > Search Guard: "20.1"
> > >
> > > ElasticSearch:6.1.1
> > >
> > >
> > >
> > >
> > >
> > >
> > > --
> > > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > > To post to this group, send email to search...@googlegroups.com.
> > > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
> > > For more options, visit https://groups.google.com/d/optout\.
> >
> >
> > --
> > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > To post to this group, send email to search...@googlegroups.com.
> > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/3ba4a31b-2667-4b82-89e7-c95fcdb98caf%40googlegroups.com\.
> > For more options, visit https://groups.google.com/d/optout\.
>
>
> --
> You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> To post to this group, send email to search...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/554d479e-fdd9-4357-ba3d-32a2d03d2699%40googlegroups.com\.
> For more options, visit https://groups.google.com/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/0718d16a-a61e-46b2-8053-fdd1b2f25ddf%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.

16

Sorry to jump on the bandwagon, but I seem to be having the exact same issue. My kibana.yml and sg_config have the same settings as the OP. I’m seeing the same xff error in the logs. When I curl directly to the elasticsearch port I’m also getting a 401.

···

On Thursday, February 1, 2018 at 1:41:37 PM UTC-8, Search Guard wrote:

Have you https enabled? If not its

curl -sS http://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.

curl --version

Am 01.02.2018 um 22:27 schrieb Kalada kalada...@t1cg.com:

This is the messages I received

  • Trying 172.22.0.3…
  • TCP_NODELAY set
  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)
  • ALPN, offering http/1.1
  • successfully set certificate verify locations:
  • CAfile: /etc/ssl/certs/ca-certificates.crt

CApath: none

  • TLSv1.2 (OUT), TLS handshake, Client hello (1):
  • error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
  • Closing connection 0

curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:

Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

Am 31.01.2018 um 22:36 schrieb Kalada kalada...@t1cg.com:

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,
“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:
“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36”,“accept”:"application/json, text/plain, /
",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,
“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,
“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},
“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},
“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.

17

@Michael: can you pls also post your sg_config.yml, elasticsearch.yml and the logs

···

Am 01.02.2018 um 23:37 schrieb Michael Evans <mike_e@rtzassociates.com>:

Sorry to jump on the bandwagon, but I seem to be having the exact same issue. My kibana.yml and sg_config have the same settings as the OP. I'm seeing the same xff error in the logs. When I curl directly to the elasticsearch port I'm also getting a 401.

On Thursday, February 1, 2018 at 1:41:37 PM UTC-8, Search Guard wrote:
Have you https enabled? If not its
curl -sS http://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
curl --version

> Am 01.02.2018 um 22:27 schrieb Kalada <kalada...@t1cg.com>:
>
> This is the messages I received
>
> * Rebuilt URL to: https://elasticsearch:9200/
> * Trying 172.22.0.3...
> * TCP_NODELAY set
> * Connected to elasticsearch (172.22.0.3) port 9200 (#0)
> * ALPN, offering http/1.1
> * successfully set certificate verify locations:
> * CAfile: /etc/ssl/certs/ca-certificates.crt
> CApath: none
> * TLSv1.2 (OUT), TLS handshake, Client hello (1):
> * error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
> * Closing connection 0
> curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
>
> On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
> Can you test it via curl or postman (without kibana) like
>
> curl -sSk https://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"
>
> > Am 31.01.2018 um 22:36 schrieb Kalada <kalada...@t1cg.com>:
> >
> > internalProxies: '.*' is the current setting. The headers appear to be valid please see an example below
> >
> > kibana_1 | {"type":"response","@timestamp":"2018-01-31T20:14:37Z","tags":,"pid":21,"method":"post","statusCode":200,
> > "req":{"url":"/elasticsearch/_msearch","method":"post","headers":{"es-security-runas-user":"USERNAME","x-proxy-user":"USENAME","host":
> > "kibana:5601","connection":"close","content-length":"732","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
> > AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36","accept":"application/json, text/plain, */*
> > ","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/x-ndjson",
> > "kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/app/kibana&quot;,
> > "x-forwarded-email":"EMAIL","x-forwarded-for":"172.19.0.1","x-forwarded-user":"USERNAME"},
> > "remoteAddress":"172.19.0.3","userAgent":"172.19.0.3","referer":"http://localhost:4180/app/kibana&quot;\},
> > "res":{"statusCode":200,"responseTime":18,"contentLength":9},"message":"POST /elasticsearch/_msearch 200 18ms - 9.0B"}
> >
> >
> >
> > On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
> > "xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present
> >
> > you can test ist by allowing simply all internal proxies
> >
> > internalProxies: '.*'
> >
> > and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia
> >
> > > Am 31.01.2018 um 19:35 schrieb Kalada <kalada...@t1cg.com>:
> > >
> > > Hi Jochen,
> > >
> > > Playing with the setting I noticed this message now
> > >
> > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
> > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
> > > org.elasticsearch.ElasticsearchSecurityException: xff not done
> > > at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
> > > at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
> > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
> > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
> > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
> > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>
> > >
> > > On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
> > > Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:
> > >
> > > TLS help | Security for Elasticsearch | Search Guard
> > >
> > > Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.
> > >
> > > Then please make a request via your proxy, and attach the logfiles here.
> > >
> > > On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
> > > Its still exhibiting the same behavior.
> > >
> > > On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
> > > In the xff section of sg_config, can you try to add:
> > >
> > > trustedProxies: '.*' # trust all external proxies, regex pattern
> > >
> > > And see if this solves the problem? Background:
> > >
> > > Search Guard Documentation 404 | Security for Elasticsearch | Search Guard
> > >
> > > searchguard.dynamic.http.xff. internalProxies:
> > >
> > > "Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."
> > >
> > > On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
> > > Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below
> > >
> > > # When this setting...s value is true Kibana uses the hostname specified in the server.host
> > >
> > > # setting. When the value of this setting is false, Kibana uses the hostname of the host
> > >
> > > # that connects to this Kibana instance.
> > >
> > > #elasticsearch.preserveHost: true
> > >
> > >
> > > # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
> > >
> > > # dashboards. Kibana creates a new index if the index doesn...t already exist.
> > >
> > > #kibana.index: ".kibana"
> > >
> > >
> > > # The default application to load.
> > >
> > > #kibana.defaultAppId: "discover"
> > >
> > >
> > > # Kibana is served by a back end server. This setting specifies the port to use.
> > >
> > > server.port: 5601
> > >
> > >
> > > # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
> > >
> > > # The default is 'localhost', which usually means remote machines will not be able to connect.
> > >
> > > # To allow connections from remote users, set this parameter to a non-loopback address.
> > >
> > > server.host: 0.0.0.0
> > >
> > >
> > > # Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
> > >
> > > # the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
> > >
> > > # to Kibana. This setting cannot end in a slash.
> > >
> > > server.basePath: /kibana
> > >
> > >
> > > # The maximum payload size in bytes for incoming server requests.
> > >
> > > #server.maxPayloadBytes: 1048576
> > >
> > >
> > > # The Kibana server's name. This is used for display purposes.
> > >
> > > #server.name: "your-hostname"
> > >
> > >
> > > # The URL of the Elasticsearch instance to use for all your queries.
> > >
> > > elasticsearch.url: http://elasticsearch:9200
> > >
> > > # If your Elasticsearch is protected with basic authentication, these settings provide
> > >
> > > # the username and password that the Kibana server uses to perform maintenance on the Kibana
> > >
> > > # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
> > >
> > > # is proxied through the Kibana server.
> > >
> > > elasticsearch.username: "kibana"
> > >
> > > elasticsearch.password: changeme
> > >
> > > searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"
> > >
> > >
> > > # Paths to the PEM-format SSL certificate and SSL key files, respectively. These
> > >
> > > # files enable SSL for outgoing requests from the Kibana server to the browser.
> > >
> > > #server.ssl.cert: /path/to/your/server.crt
> > >
> > > #server.ssl.key: /path/to/your/server.key
> > >
> > >
> > > # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
> > >
> > > # These files validate that your Elasticsearch backend uses the same key files.
> > >
> > > #elasticsearch.ssl.cert: /path/to/your/client.crt
> > >
> > > #elasticsearch.ssl.key: /path/to/your/client.key
> > >
> > >
> > > # Optional setting that enables you to specify a path to the PEM file for the certificate
> > >
> > > # authority for your Elasticsearch instance.
> > >
> > > elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem
> > >
> > >
> > > # To disregard the validity of SSL certificates, change this setting...s value to false.
> > >
> > > elasticsearch.ssl.verificationMode: "certificate"
> > >
> > > # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
> > >
> > > # the elasticsearch.requestTimeout setting.
> > >
> > > #elasticsearch.pingTimeout: 1500
> > >
> > >
> > > # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
> > >
> > > # must be a positive integer.
> > >
> > > #elasticsearch.requestTimeout: 30000
> > >
> > >
> > > # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
> > >
> > > # headers, set this value to (an empty list).
> > >
> > > elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]
> > >
> > > searchguard.basicauth.enabled: false
> > >
> > >
> > > # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
> > >
> > > # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
> > >
> > > #elasticsearch.customHeaders: {}
> > >
> > >
> > > # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
> > >
> > > #elasticsearch.shardTimeout: 0
> > >
> > >
> > > # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
> > >
> > > #elasticsearch.startupTimeout: 5000
> > >
> > >
> > > # Specifies the path where Kibana creates the process ID file.
> > >
> > > #pid.file: /var/run/kibana.pid
> > >
> > >
> > > # Enables you specify a file where Kibana stores log output.
> > >
> > > #logging.dest: stdout
> > >
> > >
> > > # Set the value of this setting to true to suppress all logging output.
> > >
> > > #logging.silent: false
> > >
> > >
> > > # Set the value of this setting to true to suppress all logging output other than error messages.
> > >
> > > #logging.quiet: false
> > >
> > >
> > > # Set the value of this setting to true to log all events, including system usage information
> > >
> > > # and all requests.
> > >
> > > #logging.verbose: false
> > >
> > >
> > > # Set the interval in milliseconds to sample system and process performance
> > >
> > > # metrics. Minimum is 100ms. Defaults to 5000.
> > >
> > > #ops.interval: 5000
> > >
> > >
> > > On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
> > > Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.
> > >
> > > See also here for an example:
> > >
> > > Kibana Authentication | Security for Elasticsearch | Search Guard
> > > ("SSO configuration: Whitelisting HTTP headers")
> > >
> > > On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
> > > When asking questions, please provide the following information:
> > >
> > > Hi,
> > >
> > >
> > > I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks
> > >
> > >
> > >
> > > nginx.conf
> > >
> > > http {
> > >
> > > server {
> > >
> > > listen ${NGINX_PORT};
> > >
> > > server_name localhost;
> > >
> > >
> > > location ${BASE_PATH} {
> > >
> > >
> > > proxy_pass http://kibana:5601/;
> > >
> > >
> > > proxy_set_header Authorization "Basic ${DIGEST}";
> > >
> > >
> > > proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;
> > >
> > > proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;
> > >
> > > proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;
> > >
> > >
> > > rewrite /login http://localhost:4180/oauth2/sign_in redirect;
> > >
> > > }
> > >
> > > }
> > >
> > >
> > >
> > > }
> > >
> > >
> > > events {
> > >
> > > worker_connections 1024;
> > >
> > > }
> > >
> > >
> > > Elastic search log entry
> > >
> > >
> > > elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]
> > >
> > > elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}
> > >
> > >
> > > Kibana Log Entry
> > >
> > > kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}
> > >
> > >
> > > sg_config.yml
> > >
> > >
> > > searchguard:
> > >
> > > dynamic:
> > >
> > > http:
> > >
> > > anonymous_auth_enabled: false
> > >
> > > xff:
> > >
> > > enabled: true
> > >
> > > internalProxies: '.*'
> > >
> > > remoteIpHeader: 'x-forwarded-for'
> > >
> > > proxiesHeader: 'x-forwarded-by'
> > >
> > > proxy_auth_domain:
> > >
> > > enabled: true
> > >
> > > order: 0
> > >
> > > http_authenticator:
> > >
> > > type: proxy
> > >
> > > challenge: false
> > >
> > > config:
> > >
> > > user_header: "x-proxy-user"
> > >
> > > roles_header: "x-proxy-roles"
> > >
> > > authentication_backend:
> > >
> > > type: noop
> > >
> > > authc:
> > >
> > > basic_internal_auth:
> > >
> > > enabled: true
> > >
> > > order: 1
> > >
> > > http_authenticator:
> > >
> > > type: basic
> > >
> > > challenge: true
> > >
> > > authentication_backend:
> > >
> > > type: internal
> > >
> > >
> > >
> > > sg_roles_mapping.yml
> > >
> > >
> > > admin:
> > >
> > > backendroles:
> > >
> > > - admin
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - elastic
> > >
> > > - user_name
> > >
> > > kibana_user:
> > >
> > > backendroles:
> > >
> > > - kibana_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - kibana
> > >
> > >
> > > logstash_user:
> > >
> > > backendroles:
> > >
> > > - logstash_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - logstash
> > >
> > >
> > > beats_user:
> > >
> > > backendroles:
> > >
> > > - beats_user
> > >
> > > host:
> > >
> > > - "*"
> > >
> > > users:
> > >
> > > - beats
> > >
> > >
> > >
> > > sg_action_group.yml
> > >
> > >
> > >
> > > # INDICES
> > >
> > > READ:
> > >
> > > - "indices:data/read*"
> > >
> > > WRITE:
> > >
> > > - "indices:data/write*"
> > >
> > > CRUD:
> > >
> > > - READ
> > >
> > > - WRITE
> > >
> > > CREATE_INDEX:
> > >
> > > - "indices:admin/create*"
> > >
> > > - "indices:admin/mapping/put"
> > >
> > > DELETE_INDEX:
> > >
> > > - "indices:admin/delete*"
> > >
> > > INDEX_OWNER:
> > >
> > > - CREATE_INDEX
> > >
> > > - CRUD
> > >
> > > INDEX_ALL:
> > >
> > > - "indices:*"
> > >
> > >
> > >
> > > # ELASTICSEARCH ENTRYPOINT
> > >
> > > GET_TEMPLATE:
> > >
> > > - "indices:admin/template/get"
> > >
> > > PUT_TEMPLATE:
> > >
> > > - "indices:admin/template/put"
> > >
> > > TEMPLATE_OWNER:
> > >
> > > - GET_TEMPLATE
> > >
> > > - PUT_TEMPLATE
> > >
> > > ES_INPUT:
> > >
> > > - TEMPLATE_OWNER
> > >
> > > - WRITE
> > >
> > > - MONITOR
> > >
> > >
> > >
> > > # MONITORING
> > >
> > > MONITOR:
> > >
> > > - "cluster:monitor*"
> > >
> > >
> > >
> > > # SUPER POWERS
> > >
> > > ALL:
> > >
> > >
> > > - "*"
> > >
> > >
> > >
> > > Search Guard: "20.1"
> > >
> > > ElasticSearch:6.1.1
> > >
> > >
> > >
> > >
> > >
> > >
> > > --
> > > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > > To post to this group, send email to search...@googlegroups.com.
> > > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
> > > For more options, visit https://groups.google.com/d/optout\.
> >
> >
> > --
> > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > To post to this group, send email to search...@googlegroups.com.
> > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/3ba4a31b-2667-4b82-89e7-c95fcdb98caf%40googlegroups.com\.
> > For more options, visit https://groups.google.com/d/optout\.
>
>
> --
> You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> To post to this group, send email to search...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/554d479e-fdd9-4357-ba3d-32a2d03d2699%40googlegroups.com\.
> For more options, visit https://groups.google.com/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/dd665e6f-fd1a-44f8-a8e7-1c9753cf329c%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.

18

elasticsearch v 6.1.1
kibana v 6.1.1

curl -vv -sSk ‘https://localhost:9200/’ -H “x-forwarded-user: 60762” -H “x-forwarded-groups: ic00” -H “x-forwarded-for: 127.0.0.1”

< HTTP/1.1 401 Unauthorized
< WWW-Authenticate: Basic realm=“Search Guard”
< content-type: text/plain; charset=UTF-8
< content-length: 12
<

  • Connection #0 to host localhost left intact
    Unauthorized

elasticsearch.logs (252 KB)

elasticsearch.yml (980 Bytes)

sg_config.yml (3.32 KB)

···

On Thursday, February 1, 2018 at 2:43:50 PM UTC-8, Search Guard wrote:

@Michael: can you pls also post your sg_config.yml, elasticsearch.yml and the logs

Am 01.02.2018 um 23:37 schrieb Michael Evans mik...@rtzassociates.com:

Sorry to jump on the bandwagon, but I seem to be having the exact same issue. My kibana.yml and sg_config have the same settings as the OP. I’m seeing the same xff error in the logs. When I curl directly to the elasticsearch port I’m also getting a 401.

On Thursday, February 1, 2018 at 1:41:37 PM UTC-8, Search Guard wrote:

Have you https enabled? If not its
curl -sS http://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
curl --version

Am 01.02.2018 um 22:27 schrieb Kalada kalada...@t1cg.com:

This is the messages I received

  • Trying 172.22.0.3…
  • TCP_NODELAY set
  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)
  • ALPN, offering http/1.1
  • successfully set certificate verify locations:
  • CAfile: /etc/ssl/certs/ca-certificates.crt
    CApath: none
  • TLSv1.2 (OUT), TLS handshake, Client hello (1):
  • error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
  • Closing connection 0
    curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

Am 31.01.2018 um 22:36 schrieb Kalada kalada...@t1cg.com:

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,
“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:
“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36”,“accept”:"application/json, text/plain, /
",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,
“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,
“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},
“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},
“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.

19

should be remoteIpHeader: 'x-forwarded-for' instead of remoteIpHeader: "X-Real-IP"

or

curl -vv -sSk 'https://localhost:9200/&#39; -H "x-forwarded-user: 60762" -H "x-forwarded-groups: ic00" -H "X-Real-IP: 127.0.0.1"

···

Am 02.02.2018 um 00:20 schrieb Michael Evans <mike_e@rtzassociates.com>:

elasticsearch v 6.1.1
kibana v 6.1.1

curl -vv -sSk 'https://localhost:9200/&#39; -H "x-forwarded-user: 60762" -H "x-forwarded-groups: ic00" -H "x-forwarded-for: 127.0.0.1"

< HTTP/1.1 401 Unauthorized
< WWW-Authenticate: Basic realm="Search Guard"
< content-type: text/plain; charset=UTF-8
< content-length: 12
<
* Connection #0 to host localhost left intact
Unauthorized

On Thursday, February 1, 2018 at 2:43:50 PM UTC-8, Search Guard wrote:
@Michael: can you pls also post your sg_config.yml, elasticsearch.yml and the logs

> Am 01.02.2018 um 23:37 schrieb Michael Evans <mik...@rtzassociates.com>:
>
> Sorry to jump on the bandwagon, but I seem to be having the exact same issue. My kibana.yml and sg_config have the same settings as the OP. I'm seeing the same xff error in the logs. When I curl directly to the elasticsearch port I'm also getting a 401.
>
> On Thursday, February 1, 2018 at 1:41:37 PM UTC-8, Search Guard wrote:
> Have you https enabled? If not its
> curl -sS http://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"
>
> otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
> curl --version
>
> > Am 01.02.2018 um 22:27 schrieb Kalada <kalada...@t1cg.com>:
> >
> > This is the messages I received
> >
> > * Rebuilt URL to: https://elasticsearch:9200/
> > * Trying 172.22.0.3...
> > * TCP_NODELAY set
> > * Connected to elasticsearch (172.22.0.3) port 9200 (#0)
> > * ALPN, offering http/1.1
> > * successfully set certificate verify locations:
> > * CAfile: /etc/ssl/certs/ca-certificates.crt
> > CApath: none
> > * TLSv1.2 (OUT), TLS handshake, Client hello (1):
> > * error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
> > * Closing connection 0
> > curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
> >
> > On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
> > Can you test it via curl or postman (without kibana) like
> >
> > curl -sSk https://elasticsearch:9200 -v -H "x-proxy-user: myuser" -H "x-proxy-roles: myrole" -H "x-forwarded-for: 127.0.0.1"
> >
> > > Am 31.01.2018 um 22:36 schrieb Kalada <kalada...@t1cg.com>:
> > >
> > > internalProxies: '.*' is the current setting. The headers appear to be valid please see an example below
> > >
> > > kibana_1 | {"type":"response","@timestamp":"2018-01-31T20:14:37Z","tags":,"pid":21,"method":"post","statusCode":200,
> > > "req":{"url":"/elasticsearch/_msearch","method":"post","headers":{"es-security-runas-user":"USERNAME","x-proxy-user":"USENAME","host":
> > > "kibana:5601","connection":"close","content-length":"732","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
> > > AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36","accept":"application/json, text/plain, */*
> > > ","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/x-ndjson",
> > > "kbn-version":"6.1.1","origin":"http://localhost:4180",“referer”:“http://localhost:4180/app/kibana&quot;,
> > > "x-forwarded-email":"EMAIL","x-forwarded-for":"172.19.0.1","x-forwarded-user":"USERNAME"},
> > > "remoteAddress":"172.19.0.3","userAgent":"172.19.0.3","referer":"http://localhost:4180/app/kibana&quot;\},
> > > "res":{"statusCode":200,"responseTime":18,"contentLength":9},"message":"POST /elasticsearch/_msearch 200 18ms - 9.0B"}
> > >
> > >
> > >
> > > On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
> > > "xff not done" means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present
> > >
> > > you can test ist by allowing simply all internal proxies
> > >
> > > internalProxies: '.*'
> > >
> > > and make sure your request contains a valid X-Forwarded-For header like described here X-Forwarded-For - Wikipedia
> > >
> > > > Am 31.01.2018 um 19:35 schrieb Kalada <kalada...@t1cg.com>:
> > > >
> > > > Hi Jochen,
> > > >
> > > > Playing with the setting I noticed this message now
> > > >
> > > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
> > > > [2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] 'ElasticsearchSecurityException[xff not done]' extracting credentials from proxy http authenticator
> > > > org.elasticsearch.ElasticsearchSecurityException: xff not done
> > > > at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > > at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > > at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > > at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
> > > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
> > > > at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
> > > > at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
> > > > at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > > at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > > at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:134) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:644) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:544) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:458) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
> > > > at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [netty-common-4.1.13.Final.jar:4.1.13.Final]
> > > > at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
> > > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from basic http authenticator
> > > > [2018-01-31T18:31:55,848][DEBUG][c.f.s.a.BackendRegistry ] User 'User [name=kibana, roles=[kibana_user], requestedTenant=null]' is authenticated
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.a.BackendRegistry ] sgtenant 'null'
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ### evaluate permissions for User [name=kibana, roles=[kibana_user], requestedTenant=null] on NODE-1
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested indices:admin/mappings/get from 172.19.0.4:42354
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolve [.kibana] from class org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest for action indices:admin/mappings/get
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] indicesOptions IndicesOptions[id=6, ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, expand_wildcards_closed=false, allow_aliases_to_multiple_indices=true, forbid_closed_indices=false, ignore_aliases=false]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 1 raw indices [.kibana]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] 0 requestTypes
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] No date math indices found
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Resolved [.kibana] to [.kibana]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final indices: [.kibana]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] pre final types:
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final indices: [.kibana]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] final types: [_all]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] requested resolved indextypes: [IndexType [index=.kibana, type=*]]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] mapped roles for kibana: [kibana_user]
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesInterceptorImpl] raw requestedTenant: 'null'
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] Result from privileges interceptor: null
> > > > [2018-01-31T18:31:55,849][DEBUG][c.f.s.c.PrivilegesEvaluator] ---------- evaluate sg_role: kibana_user
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Try wildcard match for ?kibana
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Wildcard match for ?kibana: [.kibana]
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] matches for ?kibana, will check now types [*]
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] match requested action [indices:admin/mappings/get] against ?kibana/*: [indices:*, indices:data/read*]
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] removed .kibana*
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] For index ?kibana remaining requested indextype:
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] found a match for 'kibana_user.?kibana', evaluate other roles
> > > > [2018-01-31T18:31:55,850][DEBUG][c.f.s.c.PrivilegesEvaluator] Added to leftovers kibana_user=>
> > > >
> > > > On Tuesday, January 30, 2018 at 3:54:36 PM UTC-5, Jochen Kressin wrote:
> > > > Then I guess we need the Elasticsearch logs on debug level to see what's going on behind the scenes. In order to enable it see here how to do it:
> > > >
> > > > TLS help | Security for Elasticsearch | Search Guard
> > > >
> > > > Word of warning: Since Kibana issues internal requests periodically you will see a lot of requests in the logs.
> > > >
> > > > Then please make a request via your proxy, and attach the logfiles here.
> > > >
> > > > On Tuesday, January 30, 2018 at 7:21:57 PM UTC+1, Kalada wrote:
> > > > Its still exhibiting the same behavior.
> > > >
> > > > On Tuesday, January 30, 2018 at 1:05:09 PM UTC-5, Jochen Kressin wrote:
> > > > In the xff section of sg_config, can you try to add:
> > > >
> > > > trustedProxies: '.*' # trust all external proxies, regex pattern
> > > >
> > > > And see if this solves the problem? Background:
> > > >
> > > > Search Guard Documentation 404 | Security for Elasticsearch | Search Guard
> > > >
> > > > searchguard.dynamic.http.xff. internalProxies:
> > > >
> > > > "Regular expression, defines the last proxy in the chain before the request is routed to the Elasticsearch cluster. If you have just one proxy, this is effectively the same as the trustedProxies setting. If the last proxy does not match the configured regular expression, an error is thrown and the request is denied."
> > > >
> > > > On Tuesday, January 30, 2018 at 6:43:56 PM UTC+1, Kalada wrote:
> > > > Thank you Jouchen for the quick response. Yes i have whitelisted the proxy headers see kibana.yml below
> > > >
> > > > # When this setting...s value is true Kibana uses the hostname specified in the server.host
> > > >
> > > > # setting. When the value of this setting is false, Kibana uses the hostname of the host
> > > >
> > > > # that connects to this Kibana instance.
> > > >
> > > > #elasticsearch.preserveHost: true
> > > >
> > > >
> > > > # Kibana uses an index in Elasticsearch to store saved searches, visualizations and
> > > >
> > > > # dashboards. Kibana creates a new index if the index doesn...t already exist.
> > > >
> > > > #kibana.index: ".kibana"
> > > >
> > > >
> > > > # The default application to load.
> > > >
> > > > #kibana.defaultAppId: "discover"
> > > >
> > > >
> > > > # Kibana is served by a back end server. This setting specifies the port to use.
> > > >
> > > > server.port: 5601
> > > >
> > > >
> > > > # Specifies the address to which the Kibana server will bind. IP addresses and host names are both valid values.
> > > >
> > > > # The default is 'localhost', which usually means remote machines will not be able to connect.
> > > >
> > > > # To allow connections from remote users, set this parameter to a non-loopback address.
> > > >
> > > > server.host: 0.0.0.0
> > > >
> > > >
> > > > # Enables you to specify a path to mount Kibana at if you are running behind a proxy. This only affects
> > > >
> > > > # the URLs generated by Kibana, your proxy is expected to remove the basePath value before forwarding requests
> > > >
> > > > # to Kibana. This setting cannot end in a slash.
> > > >
> > > > server.basePath: /kibana
> > > >
> > > >
> > > > # The maximum payload size in bytes for incoming server requests.
> > > >
> > > > #server.maxPayloadBytes: 1048576
> > > >
> > > >
> > > > # The Kibana server's name. This is used for display purposes.
> > > >
> > > > #server.name: "your-hostname"
> > > >
> > > >
> > > > # The URL of the Elasticsearch instance to use for all your queries.
> > > >
> > > > elasticsearch.url: http://elasticsearch:9200
> > > >
> > > > # If your Elasticsearch is protected with basic authentication, these settings provide
> > > >
> > > > # the username and password that the Kibana server uses to perform maintenance on the Kibana
> > > >
> > > > # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which
> > > >
> > > > # is proxied through the Kibana server.
> > > >
> > > > elasticsearch.username: "kibana"
> > > >
> > > > elasticsearch.password: changeme
> > > >
> > > > searchguard.cookie.password: "4fjNPloEVvrVj28BsVh3jI8t3pmv7Qg6"
> > > >
> > > >
> > > > # Paths to the PEM-format SSL certificate and SSL key files, respectively. These
> > > >
> > > > # files enable SSL for outgoing requests from the Kibana server to the browser.
> > > >
> > > > #server.ssl.cert: /path/to/your/server.crt
> > > >
> > > > #server.ssl.key: /path/to/your/server.key
> > > >
> > > >
> > > > # Optional settings that provide the paths to the PEM-format SSL certificate and key files.
> > > >
> > > > # These files validate that your Elasticsearch backend uses the same key files.
> > > >
> > > > #elasticsearch.ssl.cert: /path/to/your/client.crt
> > > >
> > > > #elasticsearch.ssl.key: /path/to/your/client.key
> > > >
> > > >
> > > > # Optional setting that enables you to specify a path to the PEM file for the certificate
> > > >
> > > > # authority for your Elasticsearch instance.
> > > >
> > > > elasticsearch.ssl.certificateAuthorities: /etc/elasticsearch/searchguard/ssl/ca/root-ca.pem
> > > >
> > > >
> > > > # To disregard the validity of SSL certificates, change this setting...s value to false.
> > > >
> > > > elasticsearch.ssl.verificationMode: "certificate"
> > > >
> > > > # Time in milliseconds to wait for Elasticsearch to respond to pings. Defaults to the value of
> > > >
> > > > # the elasticsearch.requestTimeout setting.
> > > >
> > > > #elasticsearch.pingTimeout: 1500
> > > >
> > > >
> > > > # Time in milliseconds to wait for responses from the back end or Elasticsearch. This value
> > > >
> > > > # must be a positive integer.
> > > >
> > > > #elasticsearch.requestTimeout: 30000
> > > >
> > > >
> > > > # List of Kibana client-side headers to send to Elasticsearch. To send *no* client-side
> > > >
> > > > # headers, set this value to (an empty list).
> > > >
> > > > elasticsearch.requestHeadersWhitelist: [ "authorization", "x-forwarded-for", "x-forwarded-by", "x-proxy-user", "x-proxy-roles" ]
> > > >
> > > > searchguard.basicauth.enabled: false
> > > >
> > > >
> > > > # Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
> > > >
> > > > # by client-side headers, regardless of the elasticsearch.requestHeadersWhitelist configuration.
> > > >
> > > > #elasticsearch.customHeaders: {}
> > > >
> > > >
> > > > # Time in milliseconds for Elasticsearch to wait for responses from shards. Set to 0 to disable.
> > > >
> > > > #elasticsearch.shardTimeout: 0
> > > >
> > > >
> > > > # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying.
> > > >
> > > > #elasticsearch.startupTimeout: 5000
> > > >
> > > >
> > > > # Specifies the path where Kibana creates the process ID file.
> > > >
> > > > #pid.file: /var/run/kibana.pid
> > > >
> > > >
> > > > # Enables you specify a file where Kibana stores log output.
> > > >
> > > > #logging.dest: stdout
> > > >
> > > >
> > > > # Set the value of this setting to true to suppress all logging output.
> > > >
> > > > #logging.silent: false
> > > >
> > > >
> > > > # Set the value of this setting to true to suppress all logging output other than error messages.
> > > >
> > > > #logging.quiet: false
> > > >
> > > >
> > > > # Set the value of this setting to true to log all events, including system usage information
> > > >
> > > > # and all requests.
> > > >
> > > > #logging.verbose: false
> > > >
> > > >
> > > > # Set the interval in milliseconds to sample system and process performance
> > > >
> > > > # metrics. Minimum is 100ms. Defaults to 5000.
> > > >
> > > > #ops.interval: 5000
> > > >
> > > >
> > > > On Tuesday, January 30, 2018 at 12:31:32 PM UTC-5, Jochen Kressin wrote:
> > > > Did you whitelist all the proxy headers in kibana.yml? Kibana has an HTTP header whitelist and will only pass headers listed there to Elasticsearch. Headers not listed will be silently discarded.
> > > >
> > > > See also here for an example:
> > > >
> > > > Kibana Authentication | Security for Elasticsearch | Search Guard
> > > > ("SSO configuration: Whitelisting HTTP headers")
> > > >
> > > > On Tuesday, January 30, 2018 at 5:23:49 PM UTC+1, Kalada wrote:
> > > > When asking questions, please provide the following information:
> > > >
> > > > Hi,
> > > >
> > > >
> > > > I was able to configure a docker setup of the ELK stack behind oauth2_proxy and nginx using search-guard as a authentication medium. Once a user is authenticated they are forwarded to kibana correctly. However it seems all users are assigned the internal kibana user and its associated role. I would like to be able to assign a user as an admin role, which to my understanding is done using the sg_roles_mapping.yml file, but it does not appear to be working correctly. I can see the headers are passed to Kibana but elasticsearch does not reflect this. I’d really appreciate any help. Thanks
> > > >
> > > >
> > > >
> > > > nginx.conf
> > > >
> > > > http {
> > > >
> > > > server {
> > > >
> > > > listen ${NGINX_PORT};
> > > >
> > > > server_name localhost;
> > > >
> > > >
> > > > location ${BASE_PATH} {
> > > >
> > > >
> > > > proxy_pass http://kibana:5601/;
> > > >
> > > >
> > > > proxy_set_header Authorization "Basic ${DIGEST}";
> > > >
> > > >
> > > > proxy_set_header es-security-runas-user ${DOLLAR}http_x_forwarded_user;
> > > >
> > > > proxy_set_header x-proxy-user ${DOLLAR}http_x_forwarded_user;
> > > >
> > > > proxy_set_header x-proxy-roles user ${DOLLAR}http_x_forwarded_user;
> > > >
> > > >
> > > > rewrite /login http://localhost:4180/oauth2/sign_in redirect;
> > > >
> > > > }
> > > >
> > > > }
> > > >
> > > >
> > > >
> > > > }
> > > >
> > > >
> > > > events {
> > > >
> > > > worker_connections 1024;
> > > >
> > > > }
> > > >
> > > >
> > > > Elastic search log entry
> > > >
> > > >
> > > > elasticsearch_1 | [2018-01-30T15:54:47,853][INFO ][c.f.s.c.PrivilegesEvaluator] No cluster-level perm match for User[name=kibana, roles=[kibana_user], requestedTenant=null] [IndexType [index=.kibana, type=*]] [Action [[indices:admin/template/put]]] [RolesChecked [kibana_user]]
> > > >
> > > > elasticsearch_1 | [2018-01-30T15:54:47,854][INFO ][c.f.s.c.PrivilegesEvaluator] No permissions for {}
> > > >
> > > >
> > > > Kibana Log Entry
> > > >
> > > > kibana_1 | {"type":"response","@timestamp":"2018-01-30T15:54:47Z","tags":,"pid":28,"method":"post","statusCode":503,"req":{"url":"/api/saved_objects/index-pattern?overwrite=false","method":"post","headers":{"es-security-runas-user":"user_name","x-proxy-user":"user_name","x-proxy-roles":"admin","host":"kibana:5601","connection":"close","content-length":"66","user-agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36","accept":"application/json, text/plain, */*","accept-encoding":"gzip, deflate, br","accept-language":"en-US,en;q=0.9","content-type":"application/json;charset=UTF-8","kbn-version":"6.1.1","origin":"http://localhost:4180”,“referer”:“http://localhost:4180/kibana/app/kibana",“x-forwarded-email”:"em...@email.com”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“user_name”},“remoteAddress”:“172.19.0.2”,“userAgent”:“172.19.0.2”,“referer”:"http://localhost:4180/kibana/app/kibana"},“res”:{“statusCode”:503,“responseTime”:32,“contentLength”:9},“message”:"POST /api/saved_objects/index-pattern?overwrite=false 503 32ms - 9.0B"}
> > > >
> > > >
> > > > sg_config.yml
> > > >
> > > >
> > > > searchguard:
> > > >
> > > > dynamic:
> > > >
> > > > http:
> > > >
> > > > anonymous_auth_enabled: false
> > > >
> > > > xff:
> > > >
> > > > enabled: true
> > > >
> > > > internalProxies: '.*'
> > > >
> > > > remoteIpHeader: 'x-forwarded-for'
> > > >
> > > > proxiesHeader: 'x-forwarded-by'
> > > >
> > > > proxy_auth_domain:
> > > >
> > > > enabled: true
> > > >
> > > > order: 0
> > > >
> > > > http_authenticator:
> > > >
> > > > type: proxy
> > > >
> > > > challenge: false
> > > >
> > > > config:
> > > >
> > > > user_header: "x-proxy-user"
> > > >
> > > > roles_header: "x-proxy-roles"
> > > >
> > > > authentication_backend:
> > > >
> > > > type: noop
> > > >
> > > > authc:
> > > >
> > > > basic_internal_auth:
> > > >
> > > > enabled: true
> > > >
> > > > order: 1
> > > >
> > > > http_authenticator:
> > > >
> > > > type: basic
> > > >
> > > > challenge: true
> > > >
> > > > authentication_backend:
> > > >
> > > > type: internal
> > > >
> > > >
> > > >
> > > > sg_roles_mapping.yml
> > > >
> > > >
> > > > admin:
> > > >
> > > > backendroles:
> > > >
> > > > - admin
> > > >
> > > > host:
> > > >
> > > > - "*"
> > > >
> > > > users:
> > > >
> > > > - elastic
> > > >
> > > > - user_name
> > > >
> > > > kibana_user:
> > > >
> > > > backendroles:
> > > >
> > > > - kibana_user
> > > >
> > > > host:
> > > >
> > > > - "*"
> > > >
> > > > users:
> > > >
> > > > - kibana
> > > >
> > > >
> > > > logstash_user:
> > > >
> > > > backendroles:
> > > >
> > > > - logstash_user
> > > >
> > > > host:
> > > >
> > > > - "*"
> > > >
> > > > users:
> > > >
> > > > - logstash
> > > >
> > > >
> > > > beats_user:
> > > >
> > > > backendroles:
> > > >
> > > > - beats_user
> > > >
> > > > host:
> > > >
> > > > - "*"
> > > >
> > > > users:
> > > >
> > > > - beats
> > > >
> > > >
> > > >
> > > > sg_action_group.yml
> > > >
> > > >
> > > >
> > > > # INDICES
> > > >
> > > > READ:
> > > >
> > > > - "indices:data/read*"
> > > >
> > > > WRITE:
> > > >
> > > > - "indices:data/write*"
> > > >
> > > > CRUD:
> > > >
> > > > - READ
> > > >
> > > > - WRITE
> > > >
> > > > CREATE_INDEX:
> > > >
> > > > - "indices:admin/create*"
> > > >
> > > > - "indices:admin/mapping/put"
> > > >
> > > > DELETE_INDEX:
> > > >
> > > > - "indices:admin/delete*"
> > > >
> > > > INDEX_OWNER:
> > > >
> > > > - CREATE_INDEX
> > > >
> > > > - CRUD
> > > >
> > > > INDEX_ALL:
> > > >
> > > > - "indices:*"
> > > >
> > > >
> > > >
> > > > # ELASTICSEARCH ENTRYPOINT
> > > >
> > > > GET_TEMPLATE:
> > > >
> > > > - "indices:admin/template/get"
> > > >
> > > > PUT_TEMPLATE:
> > > >
> > > > - "indices:admin/template/put"
> > > >
> > > > TEMPLATE_OWNER:
> > > >
> > > > - GET_TEMPLATE
> > > >
> > > > - PUT_TEMPLATE
> > > >
> > > > ES_INPUT:
> > > >
> > > > - TEMPLATE_OWNER
> > > >
> > > > - WRITE
> > > >
> > > > - MONITOR
> > > >
> > > >
> > > >
> > > > # MONITORING
> > > >
> > > > MONITOR:
> > > >
> > > > - "cluster:monitor*"
> > > >
> > > >
> > > >
> > > > # SUPER POWERS
> > > >
> > > > ALL:
> > > >
> > > >
> > > > - "*"
> > > >
> > > >
> > > >
> > > > Search Guard: "20.1"
> > > >
> > > > ElasticSearch:6.1.1
> > > >
> > > >
> > > >
> > > >
> > > >
> > > >
> > > > --
> > > > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > > > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > > > To post to this group, send email to search...@googlegroups.com.
> > > > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d04c8a95-74cb-41c7-a24d-baf2e0fcda2d%40googlegroups.com\.
> > > > For more options, visit https://groups.google.com/d/optout\.
> > >
> > >
> > > --
> > > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > > To post to this group, send email to search...@googlegroups.com.
> > > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/3ba4a31b-2667-4b82-89e7-c95fcdb98caf%40googlegroups.com\.
> > > For more options, visit https://groups.google.com/d/optout\.
> >
> >
> > --
> > You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> > To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> > To post to this group, send email to search...@googlegroups.com.
> > To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/554d479e-fdd9-4357-ba3d-32a2d03d2699%40googlegroups.com\.
> > For more options, visit https://groups.google.com/d/optout\.
>
>
> --
> You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to search-guard...@googlegroups.com.
> To post to this group, send email to search...@googlegroups.com.
> To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/dd665e6f-fd1a-44f8-a8e7-1c9753cf329c%40googlegroups.com\.
> For more options, visit https://groups.google.com/d/optout\.

--
You received this message because you are subscribed to the Google Groups "Search Guard Community Forum" group.
To unsubscribe from this group and stop receiving emails from it, send an email to search-guard+unsubscribe@googlegroups.com.
To post to this group, send email to search-guard@googlegroups.com.
To view this discussion on the web visit https://groups.google.com/d/msgid/search-guard/d6869b70-9f2e-49f6-ab7c-cf7c80bab631%40googlegroups.com\.
For more options, visit https://groups.google.com/d/optout\.
<elasticsearch.logs><elasticsearch.yml><sg_config.yml>

20

Brilliant! (You’re brilliant, not me clearly). Thank you! Now I have a separate issue but I’ll post that in a different thread.

···

On Thursday, February 1, 2018 at 3:33:52 PM UTC-8, Search Guard wrote:

should be remoteIpHeader: ‘x-forwarded-for’ instead of remoteIpHeader: “X-Real-IP”

or

curl -vv -sSk ‘https://localhost:9200/’ -H “x-forwarded-user: 60762” -H “x-forwarded-groups: ic00” -H “X-Real-IP: 127.0.0.1”

Am 02.02.2018 um 00:20 schrieb Michael Evans mik...@rtzassociates.com:

elasticsearch v 6.1.1

kibana v 6.1.1

curl -vv -sSk ‘https://localhost:9200/’ -H “x-forwarded-user: 60762” -H “x-forwarded-groups: ic00” -H “x-forwarded-for: 127.0.0.1”

< HTTP/1.1 401 Unauthorized

< WWW-Authenticate: Basic realm=“Search Guard”

< content-type: text/plain; charset=UTF-8

< content-length: 12

<

  • Connection #0 to host localhost left intact

Unauthorized

On Thursday, February 1, 2018 at 2:43:50 PM UTC-8, Search Guard wrote:

@Michael: can you pls also post your sg_config.yml, elasticsearch.yml and the logs

Am 01.02.2018 um 23:37 schrieb Michael Evans mik...@rtzassociates.com:

Sorry to jump on the bandwagon, but I seem to be having the exact same issue. My kibana.yml and sg_config have the same settings as the OP. I’m seeing the same xff error in the logs. When I curl directly to the elasticsearch port I’m also getting a 401.

On Thursday, February 1, 2018 at 1:41:37 PM UTC-8, Search Guard wrote:
Have you https enabled? If not its
curl -sS http://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

otherwise you curl library seems messed up. Make sure its compiled against a recent openssl version.
curl --version

Am 01.02.2018 um 22:27 schrieb Kalada kalada...@t1cg.com:

This is the messages I received

  • Trying 172.22.0.3…
  • TCP_NODELAY set
  • Connected to elasticsearch (172.22.0.3) port 9200 (#0)
  • ALPN, offering http/1.1
  • successfully set certificate verify locations:
  • CAfile: /etc/ssl/certs/ca-certificates.crt
    CApath: none
  • TLSv1.2 (OUT), TLS handshake, Client hello (1):
  • error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number
  • Closing connection 0
    curl: (35) error:1400410B:SSL routines:CONNECT_CR_SRVR_HELLO:wrong version number

On Thursday, February 1, 2018 at 3:42:22 PM UTC-5, Search Guard wrote:
Can you test it via curl or postman (without kibana) like

curl -sSk https://elasticsearch:9200 -v -H “x-proxy-user: myuser” -H “x-proxy-roles: myrole” -H “x-forwarded-for: 127.0.0.1”

Am 31.01.2018 um 22:36 schrieb Kalada kalada...@t1cg.com:

internalProxies: ‘.*’ is the current setting. The headers appear to be valid please see an example below

kibana_1 | {“type”:“response”,“@timestamp”:“2018-01-31T20:14:37Z”,“tags”:,“pid”:21,“method”:“post”,“statusCode”:200,
“req”:{“url”:“/elasticsearch/_msearch”,“method”:“post”,“headers”:{“es-security-runas-user”:“USERNAME”,“x-proxy-user”:“USENAME”,“host”:
“kibana:5601”,“connection”:“close”,“content-length”:“732”,“user-agent”:“Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/ADDRESS Safari/537.36”,“accept”:"application/json, text/plain, /
",“accept-encoding”:“gzip, deflate, br”,“accept-language”:“en-US,en;q=0.9”,“content-type”:“application/x-ndjson”,
“kbn-version”:“6.1.1”,“origin”:“http://localhost:4180”,“referer”:“http://localhost:4180/app/kibana”,
“x-forwarded-email”:“EMAIL”,“x-forwarded-for”:“172.19.0.1”,“x-forwarded-user”:“USERNAME”},
“remoteAddress”:“172.19.0.3”,“userAgent”:“172.19.0.3”,“referer”:“http://localhost:4180/app/kibana”},
“res”:{“statusCode”:200,“responseTime”:18,“contentLength”:9},“message”:“POST /elasticsearch/_msearch 200 18ms - 9.0B”}

On Wednesday, January 31, 2018 at 4:05:14 PM UTC-5, Search Guard wrote:
“xff not done” means typically that non of your configured internal proxies match and/or the X-Forwarded-For header is not present

you can test ist by allowing simply all internal proxies

internalProxies: ‘.*’

and make sure your request contains a valid X-Forwarded-For header like described here https://en.wikipedia.org/wiki/X-Forwarded-For

Am 31.01.2018 um 19:35 schrieb Kalada kalada...@t1cg.com:

Hi Jochen,

Playing with the setting I noticed this message now

[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] Try to extract auth creds from proxy http authenticator
[2018-01-31T18:31:55,847][DEBUG][c.f.s.a.BackendRegistry ] ‘ElasticsearchSecurityException[xff not done]’ extracting credentials from proxy http authenticator
org.elasticsearch.ElasticsearchSecurityException: xff not done
at com.floragunn.searchguard.http.HTTPProxyAuthenticator.extractCredentials(HTTPProxyAuthenticator.java:49) ~[search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.auth.BackendRegistry.authenticate(BackendRegistry.java:380) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.checkAndAuthenticateRequest(SearchGuardRestFilter.java:122) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter.access$000(SearchGuardRestFilter.java:47) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at com.floragunn.searchguard.filter.SearchGuardRestFilter$1.handleRequest(SearchGuardRestFilter.java:74) [search-guard-6-6.1.1-20.1.jar:6.1.1-20.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:240) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:336) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:174) [elasticsearch-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpServerTransport.dispatchRequest(Netty4HttpServerTransport.java:497) [transport-netty4-client-6.1.1.jar:6.1.1]
at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:80) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.pipelining.HttpPipeliningHandler.channelRead(HttpPipeliningHandler.java:68) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at org.elasticsearch.http.netty4.cors.Netty4CorsHandler.channelRead(Netty4CorsHandler.java:76) [transport-netty4-client-6.1.1.jar:6.1.1]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284) [netty-codec-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [netty-transport-4.1.13.Final.jar:4.1.13.Final]
at io.netty.channel.AbstractChannelHandlerContext.