Update sophos/xg to ECS 1.8 (elastic#23967)

Updates sophos/xg ECS mappings: - populate related.hosts. - avoid duplicates in related fields. - set user.name for authentications.
leweafan · Feb 11, 2021 · 192bb00 · 192bb00
1 parent a7a45ba
commit 192bb00
Show file tree

Hide file tree

Showing 21 changed files with 302 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -843,6 +843,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Updated aws/cloudtrail fileset to ECS 1.8. {issue}23118[23118] {pull}23911[23911]
 - Upgrade juniper/srx to ecs 1.8.0. {issue}23118[23118] {pull}23936[23936]
 - Update mysqlenterprise module to ECS 1.8. {issue}23118[23118] {pull}23978[23978]
+- Upgrade sophos/xg fileset to ECS 1.8.0. {issue}23118[23118] {pull}23967[23967]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/sophos/xg/config/config.yml b/x-pack/filebeat/module/sophos/xg/config/config.yml
@@ -27,7 +27,7 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
   - add_fields:
       target: '_conf'
       fields:

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml
@@ -315,16 +315,18 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.ip != null'
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
-
 #############
 ## Cleanup ##
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml
@@ -206,14 +206,17 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.ip != null'
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml
@@ -237,14 +237,17 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.ip != null'
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/event.yml b/x-pack/filebeat/module/sophos/xg/ingest/event.yml
@@ -151,6 +151,11 @@ processors:
     field: source.user.name
     value: '{{sophos.xg.name}}'
     if: "ctx.sophos?.xg?.name != null"
+- set:
+    field: user.name
+    value: '{{source.user.name}}'
+    ignore_empty_value: true
+    if: 'ctx.sophos?.xg?.log_subtype == "Authentication"'
 - rename:
     field: sophos.xg.usergroupname
     target_field: source.user.group.name

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/firewall.yml b/x-pack/filebeat/module/sophos/xg/ingest/firewall.yml
@@ -401,22 +401,27 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.ip != null'
 - append:
     field: related.ip
     value: '{{source.nat.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.nat?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.nat.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.nat?.ip != null'
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/idp.yml b/x-pack/filebeat/module/sophos/xg/ingest/idp.yml
@@ -203,16 +203,17 @@ processors:
 - append:
     if: 'ctx?.source?.ip != null'
     field: related.ip
-    value:
-      - '{{source.ip}}'
+    value: '{{source.ip}}'
+    allow_duplicates: false
 - append:
     if: 'ctx?.destination?.ip != null'
     field: related.ip
-    value:
-      - '{{destination.ip}}'
+    value: '{{destination.ip}}'
+    allow_duplicates: false
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/xg/ingest/pipeline.yml
@@ -198,6 +198,11 @@ processors:
             }
         }
         ctx["host"]["name"] = name;
+- append:
+    field: related.hosts
+    value: '{{host.name}}'
+    allow_duplicates: false
+    if: 'ctx.host?.name != null'
 
 #############
 ## Cleanup ##

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/sandstorm.yml b/x-pack/filebeat/module/sophos/xg/ingest/sandstorm.yml
@@ -106,14 +106,17 @@ processors:
 - append:
     field: related.ip
     value: "{{source.ip}}"
+    allow_duplicates: false
     if: "ctx.source?.ip != null"
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 - append:
     field: related.hash
     value: "{{file.hash.sha1}}"
+    allow_duplicates: false
     if: "ctx.file?.hash?.sha1 != null"
 - remove:
     field:

diff --git a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml
@@ -250,14 +250,17 @@ processors:
 - append:
     field: related.ip
     value: '{{source.ip}}'
+    allow_duplicates: false
     if: 'ctx?.source?.ip != null'
 - append:
     field: related.ip
     value: '{{destination.ip}}'
+    allow_duplicates: false
     if: 'ctx?.destination?.ip != null'
 - append:
     field: related.user
     value: "{{source.user.name}}"
+    allow_duplicates: false
     if: "ctx.source?.user?.name != null"
 
 #############

diff --git a/x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-spam.log-expected.json
@@ -32,6 +32,9 @@
         "observer.serial_number": "1234567890123456",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "my_fancy_host"
+        ],
         "server.bytes": 0,
         "server.port": 0,
         "service.type": "sophos",
@@ -104,6 +107,9 @@
         "observer.serial_number": "1234567890123457",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "some_other_host.local"
+        ],
         "server.bytes": 0,
         "server.ip": "185.8.209.194",
         "server.port": 25,
@@ -192,6 +198,9 @@
         "observer.serial_number": "1234567890123456",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "my_fancy_host"
+        ],
         "server.bytes": 0,
         "server.ip": "185.8.209.194",
         "server.port": 25,
@@ -280,6 +289,9 @@
         "observer.serial_number": "1234567890123457",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "some_other_host.local"
+        ],
         "server.bytes": 0,
         "server.ip": "185.8.209.194",
         "server.port": 25,
@@ -355,6 +367,9 @@
         "observer.serial_number": "C44313350024-P29PUA",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.233.61",
         "server.port": 25,
@@ -423,6 +438,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.234.240",
         "server.port": 25,
@@ -491,6 +509,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.17.121",
         "server.port": 25,
@@ -557,6 +578,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.16.204",
         "server.port": 25,
@@ -624,6 +648,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.17.121",
         "server.port": 25,
@@ -688,6 +715,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.17.121",
         "server.port": 25,
@@ -755,6 +785,9 @@
         "observer.serial_number": "C44313350024-P29PUA",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "server.bytes": 0,
         "server.ip": "10.198.233.61",
         "server.port": 110,

diff --git a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json
@@ -46,6 +46,9 @@
         "observer.serial_number": "1234567890123457",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "some_other_host.local"
+        ],
         "related.ip": [
             "172.16.34.24",
             "13.226.155.93"
@@ -124,6 +127,9 @@
         "observer.serial_number": "1234567890123456",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "my_fancy_host"
+        ],
         "related.ip": [
             "172.16.34.24",
             "13.226.155.18"
@@ -199,6 +205,9 @@
         "observer.serial_number": "1234567890123457",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "some_other_host.local"
+        ],
         "related.ip": [
             "82.165.194.211",
             "186.8.209.194"
@@ -284,6 +293,9 @@
         "observer.serial_number": "1234567890123456",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "my_fancy_host"
+        ],
         "related.ip": [
             "23.254.247.78",
             "185.7.209.194"
@@ -365,6 +377,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "related.ip": [
             "10.198.16.121",
             "10.198.234.240"
@@ -436,6 +451,9 @@
         "observer.serial_number": "S4000806149EE49",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "related.ip": [
             "10.198.16.121",
             "10.198.234.240"
@@ -509,6 +527,9 @@
         "observer.serial_number": "SFDemo-2df0960",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "related.ip": [
             "10.146.13.49",
             "10.8.142.181"
@@ -574,6 +595,9 @@
         "observer.serial_number": "SFDemo-2df0960",
         "observer.type": "firewall",
         "observer.vendor": "Sophos",
+        "related.hosts": [
+            "firewall.localgroup.local"
+        ],
         "related.ip": [
             "10.146.13.49",
             "10.8.142.181"