-
Notifications
You must be signed in to change notification settings - Fork 4.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Filebeat: Elasticsearch module: Audit log #7365
Changes from all commits
f51ec76
e25836e
7042cf2
71dc9a1
f00218d
7c5dc3f
17b2b5f
db37ed3
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,44 @@ | ||
- name: audit | ||
type: group | ||
description: > | ||
fields: | ||
- name: node_name | ||
description: "The name of the node" | ||
example: "v_VJhjV" | ||
type: keyword | ||
- name: layer | ||
description: "The layer from which this event originated: rest, transport or ip_filter" | ||
example: "rest" | ||
type: keyword | ||
- name: event_type | ||
description: "The type of event that occurred: anonymous_access_denied, authentication_failed, access_denied, access_granted, connection_granted, connection_denied, tampered_request, run_as_granted, run_as_denied" | ||
example: "access_granted" | ||
type: keyword | ||
- name: origin_type | ||
description: "Where the request originated: rest (request originated from a REST API request), transport (request was received on the transport channel), local_node (the local node issued the request)" | ||
example: "local_node" | ||
type: keyword | ||
- name: origin_address | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So this is the ip of the machine on which for example an Elasticsearch client is running which makes requests? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes |
||
description: "The IP address from which the request originated" | ||
example: "192.168.1.42" | ||
type: ip | ||
- name: principal | ||
description: "The principal (username) that failed authentication" | ||
example: "_anonymous" | ||
type: keyword | ||
- name: action | ||
description: "The name of the action that was executed" | ||
example: "cluster:monitor/main" | ||
type: keyword | ||
- name: uri | ||
description: "The REST endpoint URI" | ||
example: /_xpack/security/_authenticate | ||
type: keyword | ||
- name: request | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I followed the documentation names for all of the fields from Auditing. |
||
description: "The type of request that was executed" | ||
example: "ClearScrollRequest" | ||
type: keyword | ||
- name: request_body | ||
description: "The body of the request, if enabled" | ||
example: "body" | ||
type: text |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
type: log | ||
paths: | ||
{{ range $i, $path := .paths }} | ||
- {{$path}} | ||
{{ end }} | ||
exclude_files: [".gz$"] | ||
|
||
fields: | ||
service.name: "elasticsearch" | ||
fields_under_root: true |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
{ | ||
"description": "Pipeline for parsing elasticsearch audit logs", | ||
"processors": [ | ||
{ | ||
"rename": { | ||
"field": "@timestamp", | ||
"target_field": "event.created" | ||
} | ||
}, | ||
{ | ||
"grok": { | ||
"field": "message", | ||
"patterns": [ | ||
"\\[%{TIMESTAMP_ISO8601:elasticsearch.audit.timestamp}\\]\\s*(\\[%{WORD:elasticsearch.audit.node_name}\\])?\\s*\\[%{WORD:elasticsearch.audit.layer}\\]\\s*\\[%{WORD:elasticsearch.audit.event_type}\\]\\s*(origin_type\\=\\[%{WORD:elasticsearch.audit.origin_type}\\])?,?\\s*(origin_address\\=\\[%{IPORHOST:elasticsearch.audit.origin_address}\\])?,?\\s*(principal\\=\\[%{WORD:elasticsearch.audit.principal}\\])?,?\\s*(action\\=\\[%{DATA:elasticsearch.audit.action}\\])?,?\\s*?(uri=\\[%{DATA:elasticsearch.audit.uri}\\])?,?\\s*(request\\=\\[%{WORD:elasticsearch.audit.request}\\])?,?\\s*(request_body\\=\\[%{DATA:elasticsearch.audit.request_body}\\])?,?" | ||
] | ||
} | ||
}, | ||
{ | ||
"rename": { | ||
"field": "elasticsearch.audit.timestamp", | ||
"target_field": "@timestamp" | ||
} | ||
} | ||
], | ||
"on_failure" : [{ | ||
"set" : { | ||
"field" : "error.message", | ||
"value" : "{{ _ingest.on_failure_message }}" | ||
} | ||
}] | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
module_version: 1.0 | ||
|
||
var: | ||
- name: paths | ||
default: | ||
- /var/log/elasticsearch/*_access.log | ||
os.darwin: | ||
- /usr/local/elasticsearch/*_access.log | ||
os.windows: | ||
- c:/ProgramData/Elastic/Elasticsearch/logs/*_access.log | ||
|
||
ingest_pipeline: ingest/pipeline.json | ||
input: config/audit.yml |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate] | ||
[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate] | ||
[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest] | ||
[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied] origin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate] | ||
[2018-06-19T05:26:27,268] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate] | ||
[2018-06-19T05:55:26,898] [transport] [access_denied] origin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest] | ||
[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed] origin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,114 @@ | ||
[ | ||
{ | ||
"@timestamp": "2018-06-19T05:16:15,549", | ||
"elasticsearch.audit.event_type": "authentication_failed", | ||
"elasticsearch.audit.layer": "rest", | ||
"elasticsearch.audit.origin_address": "147.107.128.77", | ||
"elasticsearch.audit.principal": "i030648", | ||
"elasticsearch.audit.uri": "/_xpack/security/_authenticate", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", | ||
"offset": 0, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:07:52,304", | ||
"elasticsearch.audit.event_type": "authentication_failed", | ||
"elasticsearch.audit.layer": "rest", | ||
"elasticsearch.audit.node_name": "v_VJhjV", | ||
"elasticsearch.audit.origin_address": "172.22.0.3", | ||
"elasticsearch.audit.principal": "rado", | ||
"elasticsearch.audit.uri": "/_xpack/security/_authenticate", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", | ||
"offset": 155, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:00:15,778", | ||
"elasticsearch.audit.action": "indices:data/read/scroll/clear", | ||
"elasticsearch.audit.event_type": "access_granted", | ||
"elasticsearch.audit.layer": "transport", | ||
"elasticsearch.audit.origin_address": "192.168.1.165", | ||
"elasticsearch.audit.origin_type": "local_node", | ||
"elasticsearch.audit.principal": "_xpack_security", | ||
"elasticsearch.audit.request": "ClearScrollRequest", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", | ||
"offset": 306, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:07:45,544", | ||
"elasticsearch.audit.event_type": "anonymous_access_denied", | ||
"elasticsearch.audit.layer": "rest", | ||
"elasticsearch.audit.node_name": "v_VJhjV", | ||
"elasticsearch.audit.origin_address": "172.22.0.3", | ||
"elasticsearch.audit.uri": "/_xpack/security/_authenticate", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", | ||
"offset": 519, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:26:27,268", | ||
"elasticsearch.audit.event_type": "authentication_failed", | ||
"elasticsearch.audit.layer": "rest", | ||
"elasticsearch.audit.origin_address": "147.107.128.77", | ||
"elasticsearch.audit.principal": "N078801", | ||
"elasticsearch.audit.uri": "/_xpack/security/_authenticate", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", | ||
"offset": 654, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:55:26,898", | ||
"elasticsearch.audit.action": "cluster:monitor/main", | ||
"elasticsearch.audit.event_type": "access_denied", | ||
"elasticsearch.audit.layer": "transport", | ||
"elasticsearch.audit.origin_address": "147.107.128.77", | ||
"elasticsearch.audit.origin_type": "rest", | ||
"elasticsearch.audit.principal": "_anonymous", | ||
"elasticsearch.audit.request": "MainRequest", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", | ||
"offset": 802, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
}, | ||
{ | ||
"@timestamp": "2018-06-19T05:24:15,190", | ||
"elasticsearch.audit.event_type": "authentication_failed", | ||
"elasticsearch.audit.layer": "rest", | ||
"elasticsearch.audit.node_name": "v_VJhjV", | ||
"elasticsearch.audit.origin_address": "172.18.0.3", | ||
"elasticsearch.audit.principal": "elastic", | ||
"elasticsearch.audit.request_body": "body", | ||
"elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", | ||
"fileset.module": "elasticsearch", | ||
"fileset.name": "audit", | ||
"input.type": "log", | ||
"message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", | ||
"offset": 986, | ||
"prospector.type": "log", | ||
"service.name": "elasticsearch" | ||
} | ||
] |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this the Elasticsearch node name? If yes, this field should end up under
elasticsearch.node.name
. Like this we will be able to correlate it.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I am investigating here a bit more, because we have several options and more fields can show up. I need to run different configurations to follow up on behaviour and best approach. I will keep in mind that this field could be moved.