Added support for intel.log zeek module (#14404)

* Added support for intel.log zeek module Enrich the #14150 supporting intel.log Co-Authored-By: Arcuri Davide <dadokkio@gmail.com> * Update fields.yml Co-Authored-By: Arcuri Davide <dadokkio@gmail.com> * intel.log example intel.log Co-Authored-By: Arcuri Davide <dadokkio@gmail.com> * added default_field: false added default_field: false Co-Authored-By: Arcuri Davide <dadokkio@gmail.com> * Generate expected zeek/intel output event * Add changelog entry * Update field docs * Misc improvements Expand dots in zeek.intel.seen Parse ts value without dropping millisecond value Add event.ingested timestamp Convert ingest node pipeline to YAML Save JSON message in event.original * Updates to zeek.intel.seen Expand dots of all seen.* fields Change name of zeek.intel.seen.fa_file to zeek.intel.seen.f as documented by Zeek. * Update field docs
elastic · Nov 19, 2019 · 7ad14e6 · 7ad14e6
1 parent 02fc1c0
commit 7ad14e6
Show file tree

Hide file tree

Showing 12 changed files with 432 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -403,6 +403,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010]
 - Remove beta flag for some filebeat modules. {pull}14374[14374]
 - Add attack_pattern_kql field to MISP threat indicators. {pull}14470[14470]
+- Add fileset to the Zeek module for the intel.log. {pull}14404[14404]
 
 *Heartbeat*
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -20827,6 +20827,154 @@ type: integer
 
 --
 
+[float]
+=== intel
+
+Fields exported by the Zeek Intel log.
+
+
+
+
+*`zeek.intel.seen.indicator`*::
++
+--
+The intelligence indicator.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.indicator_type`*::
++
+--
+The type of data the indicator represents.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.host`*::
++
+--
+If the indicator type was Intel::ADDR, then this field will be present.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.conn`*::
++
+--
+If the data was discovered within a connection, the connection record should go here to give context to the data.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.where`*::
++
+--
+Where the data was discovered.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.node`*::
++
+--
+The name of the node where the match was discovered.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.uid`*::
++
+--
+If the data was discovered within a connection, the connection uid should go here to give context to the data. If the conn field is provided, this will be automatically filled out.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.seen.f`*::
++
+--
+If the data was discovered within a file, the file record should go here to provide context to the data.
+
+
+type: object
+
+--
+
+*`zeek.intel.seen.fuid`*::
++
+--
+If the data was discovered within a file, the file uid should go here to provide context to the data. If the file record f is provided, this will be automatically filled out.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.matched`*::
++
+--
+Event to represent a match in the intelligence data from data that was seen.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.sources`*::
++
+--
+Sources which supplied data for this match.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.fuid`*::
++
+--
+If a file was associated with this intelligence hit, this is the uid for the file.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.file_mime_type`*::
++
+--
+A mime type if the intelligence hit is related to a file. If the $f field is provided this will be automatically filled out.
+
+
+type: keyword
+
+--
+
+*`zeek.intel.file_desc`*::
++
+--
+Frequently files can be described to give a bit more context. If the $f field is provided this field will be automatically filled out.
+
+
+type: keyword
+
+--
+
 [float]
 === irc
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -627,6 +627,8 @@ filebeat.modules:
     enabled: true
   http:
     enabled: true
+  intel:
+    enabled: true    
   irc:
     enabled: true
   kerberos:

diff --git a/x-pack/filebeat/module/zeek/_meta/config.yml b/x-pack/filebeat/module/zeek/_meta/config.yml
@@ -19,6 +19,8 @@
     enabled: true
   http:
     enabled: true
+  intel:
+    enabled: true    
   irc:
     enabled: true
   kerberos:

diff --git a/x-pack/filebeat/module/zeek/fields.go b/x-pack/filebeat/module/zeek/fields.go
diff --git a/x-pack/filebeat/module/zeek/intel/_meta/fields.yml b/x-pack/filebeat/module/zeek/intel/_meta/fields.yml
@@ -0,0 +1,80 @@
+- name: intel
+  type: group
+  default_field: false
+  description: >
+    Fields exported by the Zeek Intel log.
+  fields:
+
+    - name: seen
+      type: group
+      fields:
+        - name: indicator
+          type: keyword
+          description: >
+            The intelligence indicator.
+
+        - name: indicator_type
+          type: keyword
+          description: >
+            The type of data the indicator represents.
+
+        - name: host
+          type: keyword
+          description: >
+            If the indicator type was Intel::ADDR, then this field will be present.
+
+        - name: conn
+          type: keyword
+          description: >
+            If the data was discovered within a connection, the connection record should go here to give context to the data.
+
+        - name: where
+          type: keyword
+          description: >
+            Where the data was discovered.
+
+        - name: node
+          type: keyword
+          description: >
+            The name of the node where the match was discovered.
+
+        - name: uid
+          type: keyword
+          description: >
+            If the data was discovered within a connection, the connection uid should go here to give context to the data. If the conn field is provided, this will be automatically filled out.
+
+        - name: f
+          type: object
+          description: >
+            If the data was discovered within a file, the file record should go here to provide context to the data.
+
+        - name: fuid
+          type: keyword
+          description: >
+            If the data was discovered within a file, the file uid should go here to provide context to the data. If the file record f is provided, this will be automatically filled out.
+
+
+    - name: matched
+      type: keyword
+      description: >
+        Event to represent a match in the intelligence data from data that was seen.
+
+    - name: sources
+      type: keyword
+      description: >
+        Sources which supplied data for this match.
+
+    - name: fuid
+      type: keyword
+      description: >
+        If a file was associated with this intelligence hit, this is the uid for the file.
+
+    - name: file_mime_type
+      type: keyword
+      description: >
+        A mime type if the intelligence hit is related to a file. If the $f field is provided this will be automatically filled out.
+
+    - name: file_desc
+      type: keyword
+      description: >
+        Frequently files can be described to give a bit more context. If the $f field is provided this field will be automatically filled out.
diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml
@@ -0,0 +1,63 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+ - {{$path}}
+{{ end }}
+exclude_files: [".gz$"]
+tags: {{.tags}}
+
+processors:
+  - rename:
+      fields:
+        - {from: message, to: event.original}
+  - decode_json_fields:
+      fields: [event.original]
+      target: zeek.intel
+  - timestamp:
+      field: zeek.intel.ts
+      layouts: [UNIX]
+      test: "1573030980.989353"
+  - convert:
+      ignore_missing: true
+      fields:
+        - {from: zeek.intel.id.orig_h, to: source.address}
+        - {from: zeek.intel.id.orig_h, to: source.ip, type: ip}
+        - {from: zeek.intel.id.orig_p, to: source.port, type: long}
+        - {from: zeek.intel.id.resp_h, to: destination.address}
+        - {from: zeek.intel.id.resp_h, to: destination.ip, type: ip}
+        - {from: zeek.intel.id.resp_p, to: destination.port, type: long}
+  - rename:
+      ignore_missing: true
+      fields:
+        - from: zeek.intel.uid
+          to: zeek.session_id
+
+        # Expand field names containing dots.
+        - from: zeek.intel.seen.indicator
+          to: seen.indicator
+        - from: zeek.intel.seen.indicator_type
+          to: seen.indicator_type
+        - from: zeek.intel.seen.host
+          to: seen.host
+        - from: zeek.intel.seen.where
+          to: seen.where
+        - from: zeek.intel.seen.node
+          to: seen.node
+        - from: zeek.intel.seen.conn
+          to: seen.conn
+        - from: zeek.intel.seen.uid
+          to: seen.uid
+        - from: zeek.intel.seen.f
+          to: seen.f
+        - from: zeek.intel.seen.fuid
+          to: seen.fuid
+        - from: seen
+          to: zeek.intel.seen
+  - drop_fields:
+      ignore_missing: true
+      fields:
+        - zeek.intel.ts
+        - zeek.intel.id.orig_h
+        - zeek.intel.id.orig_p
+        - zeek.intel.id.resp_h
+        - zeek.intel.id.resp_p
diff --git a/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/intel/ingest/pipeline.yml
@@ -0,0 +1,72 @@
+---
+description: Pipeline for normalizing Zeek intel.log.
+processors:
+  - set:
+      field: event.created
+      value: "{{_ingest.timestamp}}"
+
+  # IP Geolocation Lookup
+  - geoip:
+      if: ctx.source?.geo == null
+      field: source.ip
+      target_field: source.geo
+      ignore_missing: true
+      properties:
+        - city_name
+        - continent_name
+        - country_iso_code
+        - country_name
+        - location
+        - region_iso_code
+        - region_name
+  - geoip:
+      if: ctx.destination?.geo == null
+      field: destination.ip
+      target_field: destination.geo
+      ignore_missing: true
+      properties:
+        - city_name
+        - continent_name
+        - country_iso_code
+        - country_name
+        - location
+        - region_iso_code
+        - region_name
+
+  # IP Autonomous System (AS) Lookup
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: source.ip
+      target_field: source.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - geoip:
+      database_file: GeoLite2-ASN.mmdb
+      field: destination.ip
+      target_field: destination.as
+      properties:
+        - asn
+        - organization_name
+      ignore_missing: true
+  - rename:
+      field: source.as.asn
+      target_field: source.as.number
+      ignore_missing: true
+  - rename:
+      field: source.as.organization_name
+      target_field: source.as.organization.name
+      ignore_missing: true
+  - rename:
+      field: destination.as.asn
+      target_field: destination.as.number
+      ignore_missing: true
+  - rename:
+      field: destination.as.organization_name
+      target_field: destination.as.organization.name
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{ _ingest.on_failure_message }}"