Upgrade fortinet/firewall to ECS 1.8 (#23902)

CHANGELOG.next.asciidoc

@@ -834,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Updated microsoft defender_atp and m365_defender to ECS 1.8. {pull}23897[23897] {issue}23118[23118]
 - Updated o365 module to ECS 1.8. {issue}23118[23118] {pull}23896[23896]
 - Upgrade CEF module to ECS 1.8.0. {pull}23832[23832]
+- Upgrade fortinet/firewall to ECS 1.8 {issue}23118[23118] {pull}23902[23902]
 
 *Heartbeat*
 

x-pack/filebeat/module/fortinet/firewall/config/firewall.yml

@@ -27,7 +27,7 @@ processors:
   - add_fields:
       target: ''
       fields:
-        ecs.version: 1.7.0
+        ecs.version: 1.8.0
 
 {{ if .external_interfaces }}
   - add_fields:

x-pack/filebeat/module/fortinet/firewall/ingest/event.yml

@@ -242,93 +242,6 @@ processors:
     type: integer
     ignore_failure: true
     ignore_missing: true
-- geoip:
-    field: source.ip
-    target_field: source.geo
-    ignore_missing: true
-    if: "ctx.source?.geo == null"
-- geoip:
-    field: destination.ip
-    target_field: destination.geo
-    ignore_missing: true
-    if: "ctx.destination?.geo == null"
-- geoip:
-    database_file: GeoLite2-ASN.mmdb
-    field: source.ip
-    target_field: source.as
-    properties:
-    - asn
-    - organization_name
-    ignore_missing: true
-- geoip:
-    database_file: GeoLite2-ASN.mmdb
-    field: destination.ip
-    target_field: destination.as
-    properties:
-    - asn
-    - organization_name
-    ignore_missing: true
-- geoip:
-    field: source.nat.ip
-    target_field: source.geo
-    ignore_missing: true
-    if: "ctx.source?.geo == null"
-- geoip:
-    field: destination.nat.ip
-    target_field: destination.geo
-    ignore_missing: true
-    if: "ctx.destination?.geo == null"
-- geoip:
-    database_file: GeoLite2-ASN.mmdb
-    field: source.nat.ip
-    target_field: source.as
-    properties:
-    - asn
-    - organization_name
-    ignore_missing: true
-    if: "ctx.source?.as == null"
-- geoip:
-    database_file: GeoLite2-ASN.mmdb
-    field: destination.nat.ip
-    target_field: destination.as
-    properties:
-    - asn
-    - organization_name
-    ignore_missing: true
-    if: "ctx.destination?.as == null"
-- rename:
-    field: source.as.asn
-    target_field: source.as.number
-    ignore_missing: true
-- rename:
-    field: source.as.organization_name
-    target_field: source.as.organization.name
-    ignore_missing: true
-- rename:
-    field: destination.as.asn
-    target_field: destination.as.number
-    ignore_missing: true
-- rename:
-    field: destination.as.organization_name
-    target_field: destination.as.organization.name
-    ignore_missing: true
-- script:
-    lang: painless
-    source: ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes
-    if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null"
-    ignore_failure: true
-- append:
-    field: related.ip
-    value: "{{source.ip}}"
-    if: "ctx.source?.ip != null"
-- append:
-    field: related.ip
-    value: "{{destination.ip}}"
-    if: "ctx.destination?.ip != null"
-- append:
-    field: related.user
-    value: "{{source.user.name}}"
-    if: "ctx.source?.user?.name != null"
 - remove:
     field:
     - fortinet.firewall.dstport

x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml

@@ -15,14 +15,17 @@ processors:
     ignore_missing: true
     ignore_failure: false
     trim_value: "\""
-- remove:
-    field: fortinet.tmp.assignip
-    if: "ctx.fortinet?.tmp?.assignip == 'N/A'"
-    ignore_missing: true
 - rename:
     field: fortinet.tmp
     target_field: fortinet.firewall
     ignore_missing: true
+- script:
+    lang: painless
+    source: |
+      def fw = ctx?.fortinet?.firewall;
+      if (fw != null) {
+        fw.entrySet().removeIf(entry -> entry.getValue() == "N/A");
+      }
 - set:
     field: observer.vendor
     value: Fortinet
@@ -134,36 +137,6 @@ processors:
     field: fortinet.firewall.level
     target_field: log.level
     ignore_missing: true
-- remove:
-    field: fortinet.firewall.assignip
-    if: "ctx.fortinet?.firewall?.assignip == 'N/A'"
-- remove:
-    field: fortinet.firewall.dstip
-    if: "ctx.fortinet?.firewall?.dstip == 'N/A'"
-- remove:
-    field: fortinet.firewall.srcip
-    if: "ctx.fortinet?.firewall?.srcip == 'N/A'"
-- remove:
-    field: fortinet.firewall.remip
-    if: "ctx.fortinet?.firewall?.remip == 'N/A'"
-- remove:
-    field: fortinet.firewall.locip
-    if: "ctx.fortinet?.firewall?.locip == 'N/A'"
-- remove:
-    field: fortinet.firewall.group
-    if: "ctx.fortinet?.firewall?.group == 'N/A'"
-- remove:
-    field: fortinet.firewall.user
-    if: "ctx.fortinet?.firewall?.user == 'N/A'"
-- remove:
-    field: fortinet.firewall.tranip
-    if: "ctx.fortinet?.firewall?.tranip == 'N/A'"
-- remove:
-    field: fortinet.firewall.transip
-    if: "ctx.fortinet?.firewall?.transip == 'N/A'"
-- remove:
-    field: fortinet.firewall.tunnelip
-    if: "ctx.fortinet?.firewall?.tunnelip == 'N/A'"
 # Handle interface-based network directionality
 - set:
     field: network.direction
@@ -259,6 +232,128 @@ processors:
     field: fortinet.firewall.size
     type: long
     ignore_missing: true
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_missing: true
+    if: "ctx.source?.geo == null"
+- geoip:
+    field: destination.ip
+    target_field: destination.geo
+    ignore_missing: true
+    if: "ctx.destination?.geo == null"
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: destination.ip
+    target_field: destination.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- geoip:
+    field: source.nat.ip
+    target_field: source.geo
+    ignore_missing: true
+    if: "ctx.source?.geo == null"
+- geoip:
+    field: destination.nat.ip
+    target_field: destination.geo
+    ignore_missing: true
+    if: "ctx.destination?.geo == null"
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.nat.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+    if: "ctx.source?.as == null"
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: destination.nat.ip
+    target_field: destination.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+    if: "ctx.destination?.as == null"
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- rename:
+    field: destination.as.asn
+    target_field: destination.as.number
+    ignore_missing: true
+- rename:
+    field: destination.as.organization_name
+    target_field: destination.as.organization.name
+    ignore_missing: true
+- script:
+    lang: painless
+    source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes"
+    if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null"
+    ignore_failure: true
+- script:
+    lang: painless
+    source: "ctx.network.packets = ctx.source.packets + ctx.destination.packets"
+    if: "ctx?.source?.packets != null && ctx?.destination?.packets != null"
+    ignore_failure: true
+- append:
+    field: related.ip
+    value: "{{source.ip}}"
+    if: "ctx.source?.ip != null"
+- append:
+    field: related.ip
+    value: "{{destination.ip}}"
+    if: "ctx.destination?.ip != null"
+- append:
+    field: related.user
+    value: "{{source.user.name}}"
+    if: "ctx.source?.user?.name != null"
+- append:
+    field: related.user
+    value: "{{destination.user.name}}"
+    if: "ctx.destination?.user?.name != null"
+- append:
+    field: related.hosts
+    value: "{{destination.address}}"
+    if: "ctx.destination?.address != null"
+- append:
+    field: related.hosts
+    value: "{{source.address}}"
+    if: "ctx.source?.address != null"
+- append:
+    field: related.hosts
+    value: "{{dns.question.name}}"
+    if: "ctx.dns?.question?.name != null"
+- script:
+    lang: painless
+    source: |
+      def dnsIPs = ctx?.dns?.resolved_ip;
+      if (dnsIPs != null && dnsIPs instanceof List) {
+        if (ctx?.related?.ip == null) {
+          ctx.related.ip = [];
+        }
+        for (ip in dnsIPs) {
+          if (!ctx.related.ip.contains(ip)) {
+            ctx.related.ip.add(ip);
+          }
+        }
+      }
 on_failure:
 - set:
     field: error.message