Upgrade elasticsearch/audit to ECS 1.8 (elastic#24000)

CHANGELOG.next.asciidoc

@@ -845,6 +845,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update mysqlenterprise module to ECS 1.8. {issue}23118[23118] {pull}23978[23978]
 - Upgrade sophos/xg fileset to ECS 1.8.0. {issue}23118[23118] {pull}23967[23967]
 - Upgrade system/auth to ECS 1.8 {issue}23118[23118] {pull}23961[23961]
+- Upgrade elasticsearch/audit to ECS 1.8 {issue}23118[23118] {pull}24000[24000]
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -47765,11 +47765,6 @@ example: 0
 
 --
 
-[float]
-=== audit
-
-
-
 
 *`elasticsearch.audit.layer`*::
 +
@@ -47833,6 +47828,27 @@ example: ['kibana_admin', 'beats_admin']
 
 --
 
+*`elasticsearch.audit.user.run_as.name`*::
++
+--
+type: keyword
+
+--
+
+*`elasticsearch.audit.user.run_as.realm`*::
++
+--
+type: keyword
+
+--
+
+*`elasticsearch.audit.component`*::
++
+--
+type: keyword
+
+--
+
 *`elasticsearch.audit.action`*::
 +
 --
@@ -47929,6 +47945,13 @@ type: text
 
 --
 
+*`elasticsearch.audit.invalidate.apikeys.owned_by_authenticated_user`*::
++
+--
+type: boolean
+
+--
+
 [float]
 === deprecation
 

filebeat/module/elasticsearch/audit/_meta/fields.yml

@@ -1,6 +1,5 @@
 - name: audit
   type: group
-  description: >
   fields:
     - name: layer
       description: "The layer from which this event originated: rest, transport or ip_filter"
@@ -26,6 +25,12 @@
       description: "Roles to which the principal belongs"
       example: [ "kibana_admin", "beats_admin" ]
       type: keyword
+    - name: user.run_as.name
+      type: keyword
+    - name: user.run_as.realm
+      type: keyword
+    - name: component
+      type: keyword
     - name: action
       description: "The name of the action that was executed"
       example: "cluster:monitor/main"
@@ -63,3 +68,5 @@
       migration: true
     - name: message
       type: text
+    - name: invalidate.apikeys.owned_by_authenticated_user
+      type: boolean

filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml

@@ -3,8 +3,6 @@ processors:
 - json:
     field: message
     target_field: elasticsearch.audit
-- drop:
-    if: ctx.elasticsearch.audit?.type != null && ctx.elasticsearch.audit.type != 'audit'
 - remove:
     field: elasticsearch.audit.type
     ignore_missing: true
@@ -16,6 +14,7 @@ processors:
     - yyyy-MM-dd'T'HH:mm:ss,SSS
     - yyyy-MM-dd'T'HH:mm:ss,SSSZ
     timezone: '{{ event.timezone }}'
+    ignore_failure: true
 - remove:
     if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null
     field: event.timezone
@@ -80,13 +79,64 @@ processors:
 - rename:
     field: elasticsearch.audit.node
     target_field: elasticsearch.node
+- rename:
+    field: elasticsearch.audit.change.disable.user.name
+    target_field: user.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.change.enable.user.name
+    target_field: user.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.delete.user.name
+    target_field: user.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.put.user.name
+    target_field: user.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.put.user.full_name
+    target_field: user.full_name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.put.user.email
+    target_field: user.email
+    ignore_missing: true
+- remove:
+    field: elasticsearch.audit.put
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.invalidate.apikeys.user.name
+    target_field: user.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.audit.invalidate.apikeys.user.realm
+    target_field: elasticsearch.audit.user.realm
+    ignore_missing: true
+- dot_expander:
+    field: user.run_as.name
+    path: elasticsearch.audit
+    ignore_failure: true
+- dot_expander:
+    field: user.run_as.realm
+    path: elasticsearch.audit
+    ignore_failure: true
+- convert:
+    field: elasticsearch.audit.user.run_as.name
+    target_field: user.effective.name
+    type: string
+    ignore_failure: true
 - dot_expander:
     field: user.name
     path: elasticsearch.audit
 - rename:
     field: elasticsearch.audit.user.name
     target_field: user.name
     ignore_missing: true
+- dot_expander:
+    field: user.email
+    path: elasticsearch.audit
 - dot_expander:
     field: request.method
     path: elasticsearch.audit
@@ -104,10 +154,17 @@ processors:
 - dot_expander:
     field: cluster.name
     path: elasticsearch.audit
+- dot_expander:
+    field: cluster.uuid
+    path: elasticsearch.audit
 - rename:
     field: elasticsearch.audit.cluster.name
     target_field: elasticsearch.cluster.name
     ignore_missing: true
+- rename:
+    field: elasticsearch.audit.cluster.uuid
+    target_field: elasticsearch.cluster.uuid
+    ignore_missing: true
 - rename:
     field: elasticsearch.audit.level
     target_field: log.level

filebeat/module/elasticsearch/audit/ingest/pipeline.yml

@@ -55,6 +55,10 @@ processors:
     field: related.user
     value: "{{user.name}}"
     if: "ctx?.user?.name != null"
+- append:
+    field: related.user
+    value: "{{user.effective.name}}"
+    if: "ctx?.user?.effective?.name != null"
 - remove:
     field: elasticsearch.audit.@timestamp
 - remove:

filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json

@@ -23,6 +23,27 @@
         "source.port": 40380,
         "url.original": "/"
     },
+    {
+        "@timestamp": "2019-06-11T15:03:32.777Z",
+        "elasticsearch.audit.component": "o.e.x.s.a.AuthenticationService",
+        "elasticsearch.audit.message": "Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]",
+        "elasticsearch.cluster.name": "docker-cluster",
+        "elasticsearch.cluster.uuid": "xEiKc6ipRiyzU8_8czXrJw",
+        "elasticsearch.node.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
+        "elasticsearch.node.name": "dff7befc418f",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.audit",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.outcome": "failure",
+        "fileset.name": "audit",
+        "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
+        "input.type": "log",
+        "log.level": "INFO",
+        "log.offset": 299,
+        "message": "{\"type\": \"server\", \"timestamp\": \"2019-06-11T15:03:32,777+0000\", \"level\": \"INFO\", \"component\": \"o.e.x.s.a.AuthenticationService\", \"cluster.name\": \"docker-cluster\", \"node.name\": \"dff7befc418f\", \"cluster.uuid\": \"xEiKc6ipRiyzU8_8czXrJw\", \"node.id\": \"Xaq2BFVcQ1OhyMrjL8gNOg\",  \"message\": \"Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]\"  }",
+        "service.type": "elasticsearch"
+    },
     {
         "@timestamp": "2019-06-11T15:03:32.778Z",
         "elasticsearch.audit.layer": "rest",

filebeat/module/elasticsearch/audit/test/test-audit.log

@@ -5,3 +5,10 @@
 {"@timestamp":"2018-10-31T09:35:12,303", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest","origin.address":"[::1]:61711", "action":"cluster:admin/xpack/security/user/change_password", "request.name":"ChangePasswordRequest"}
 {"@timestamp":"2018-10-31T09:35:12,314", "node.id":"DSiWcTyeThWtUXLB9J0BMw", "event.type":"transport", "event.action":"access_granted", "user.name":"_xpack_security", "user.realm":"__attach", "user.roles":["superuser"], "origin.type":"local_node", "origin.address":"127.0.0.1:9300", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":[".security-6"]}
 {"@timestamp":"2019-01-27T20:15:10,380", "node.name":"node-0", "node.id":"y8fa3M5zSSGo1M_KJRMUXw", "event.type":"rest", "event.action":"authentication_success", "user.name":"elastic-admin", "origin.type":"rest", "origin.address":"[::1]:58955", "realm":"default_file", "url.path":"/_search", "request.method":"GET", "request.body":"\n{\n    \"query\" : {\n        \"term\" : { \"user\" : \"kimchy\" }\n    }\n}\n", "request.id":"WzL_kb6VSvOhAq0twPvHOQ"}
+{"type":"audit", "timestamp":"2020-12-30T23:17:28,308+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"security_config_change", "event.action":"change_disable_user", "request.id":"qvLIgw_eTvyK3cgV-GaLVg", "change":{"disable":{"user":{"name":"user1"}}}}
+{"type":"audit", "timestamp":"2020-12-30T23:17:34,843+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"security_config_change", "event.action":"change_enable_user", "request.id":"BO3QU3qeTb-Ei0G0rUOalQ", "change":{"enable":{"user":{"name":"user1"}}}}
+{"type":"audit", "timestamp":"2020-12-30T22:19:41,345+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"security_config_change", "event.action":"delete_user", "request.id":"au5a1Cc3RrebDMitMGGNCw", "delete":{"user":{"name":"jacknich"}}}
+{"type":"audit", "timestamp":"2020-12-31T00:36:30,247+0200", "node.id":"9clhpgjJRR-iKzOw20xBNQ", "event.type":"security_config_change", "event.action":"invalidate_apikeys", "request.id":"7lyIQU9QTFqSrTxD0CqnTQ", "invalidate":{"apikeys":{"owned_by_authenticated_user":false,"user":{"name":"myuser","realm":"native1"}}}}
+{"type":"audit", "timestamp":"2020-12-30T22:10:09,749+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"security_config_change", "event.action":"put_user", "request.id":"VIiSvhp4Riim_tpkQCVSQA", "put":{"user":{"name":"user1","enabled":false,"roles":["admin","other_role1"],"full_name":"Jack Sparrow","email":"jack@blackpearl.com","has_password":true,"metadata":{"cunning":10}}}}
+{"type":"audit", "timestamp":"2020-12-30T22:49:34,859+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"transport", "event.action":"run_as_denied", "user.name":"user1", "user.run_as.name":"user1", "user.realm":"default_native", "user.run_as.realm":"default_native", "user.roles":["test_role"], "origin.type":"rest", "origin.address":"[::1]:52662", "request.id":"RcaSt872RG-R_WJBEGfYXA", "action":"indices:data/read/search", "request.name":"SearchRequest", "indices":["alias1"]}
+{"type":"audit", "timestamp":"2020-12-30T22:44:42,068+0200", "node.id":"0RMNyghkQYCc_gVd1G6tZQ", "event.type":"transport", "event.action":"run_as_granted", "user.name":"elastic", "user.run_as.name":"user1", "user.realm":"reserved", "user.run_as.realm":"default_native", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:52623", "request.id":"dGqPTdEQSX2TAPS3cvc1qA", "action":"indices:data/read/search", "request.name":"SearchRequest", "indices":["alias1"]}