Update Microsoft module to ECS 1.8

CHANGELOG.next.asciidoc

@@ -843,6 +843,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add new ECS user and categories features to google_workspace/gsuite {issue}23118[23118] {pull}23709[23709]
 - Move crowdstrike JS processor to ingest pipelines and upgrade to ECS 1.8.0 {issue}23118[23118] {pull}23875[23875]
 - Update Filebeat auditd dataset to ECS 1.8.0. {pull}23723[23723] {issue}23118[23118]
+- Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 
 *Heartbeat*
 

x-pack/filebeat/module/microsoft/defender_atp/config/atp.yml

@@ -54,4 +54,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0

x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml

@@ -249,17 +249,34 @@ processors:
 ######################
 - rename:
     field: json.relatedUser.userName
-    target_field: host.user.name
+    target_field: user.name
     ignore_missing: true
 - rename:
     field: json.relatedUser.domainName
-    target_field: host.user.domain
+    target_field: user.domain
     ignore_missing: true
 - rename:
     field: json.evidence.userSid
-    target_field: host.user.id
+    target_field: user.id
     ignore_missing: true
 
+##############################
+##   ECS host.user Mapping  ##
+## Deprecated since ECS 1.8 ##
+##############################
+- set:
+    field: host.user.name
+    value: '{{user.name}}'
+    ignore_empty_value: true
+- set:
+    field: host.user.domain
+    value: '{{user.domain}}'
+    ignore_empty_value: true
+- set:
+    field: host.user.id
+    value: '{{user.id}}'
+    ignore_empty_value: true
+
 #########################
 ## ECS Related Mapping ##
 #########################
@@ -269,8 +286,8 @@ processors:
     if: ctx.json?.evidence?.ipAddress != null
 - append:
     field: related.user
-    value: '{{host.user.name}}'
-    if: ctx.host?.user?.name != null
+    value: '{{user.name}}'
+    if: ctx.user?.name != null
 - append:
     field: related.hash
     value: '{{file.hash.sha1}}'

x-pack/filebeat/module/microsoft/defender_atp/test/defender_atp-test.json.log-expected.json

@@ -118,7 +118,9 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "DefenseEvasion"
+        "threat.technique.name": "DefenseEvasion",
+        "user.domain": "TestServer4",
+        "user.name": "administrator1"
     },
     {
         "cloud.account.id": "43521344-d66c-4c7e-9e30-40034eb7c6f3",
@@ -176,7 +178,10 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "DefenseEvasion"
+        "threat.technique.name": "DefenseEvasion",
+        "user.domain": "TestServer4",
+        "user.id": "S-1-5-21-46152456-1367606905-4031241297-500",
+        "user.name": "administrator1"
     },
     {
         "cloud.account.id": "1234543-d66c-4c7e-9e30-40034eb7c6f3",

x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml

@@ -54,4 +54,4 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0

x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml

@@ -227,17 +227,34 @@ processors:
 ######################
 - rename:
     field: json.alerts.entities.userPrincipalName
-    target_field: host.user.name
+    target_field: user.name
     ignore_missing: true
 - rename:
     field: json.alerts.entities.domainName
-    target_field: host.user.domain
+    target_field: user.domain
     ignore_missing: true
 - rename:
     field: json.alerts.entities.aadUserId
-    target_field: host.user.id
+    target_field: user.id
     ignore_missing: true
 
+##############################
+##   ECS host.user Mapping  ##
+## Deprecated since ECS 1.8 ##
+##############################
+- set:
+    field: host.user.name
+    value: '{{user.name}}'
+    ignore_empty_value: true
+- set:
+    field: host.user.domain
+    value: '{{user.domain}}'
+    ignore_empty_value: true
+- set:
+    field: host.user.id
+    value: '{{user.id}}'
+    ignore_empty_value: true
+
 #########################
 ## ECS Related Mapping ##
 #########################
@@ -247,8 +264,8 @@ processors:
     if: ctx.json?.entities?.ipAddress != null
 - append:
     field: related.user
-    value: '{{host.user.name}}'
-    if: ctx.host?.user?.name != null
+    value: '{{user.name}}'
+    if: ctx.user?.name != null
 - append:
     field: related.hash
     value: '{{file.hash.sha1}}'

x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json

@@ -556,7 +556,9 @@
             "forwarded"
         ],
         "threat.framework": "MITRE ATT&CK",
-        "threat.technique.name": "SuspiciousActivity"
+        "threat.technique.name": "SuspiciousActivity",
+        "user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3",
+        "user.name": "brent@elasticbv.onmicrosoft.com"
     },
     {
         "@timestamp": "2020-09-23T19:32:05.8366667Z",