From 59a05ed7c8905f4e0d39517a34d977680051055c Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Fri, 22 Feb 2019 16:46:16 -0500 Subject: [PATCH] Create module for NetFlow to add geoip (#10877) This creates a module around the Filebeat netflow input in order to enrich the events with source.geo.* and destination.geo.* fields by using the geoip processor in Elasticsearch Ingest Node. Usage: ``` filebeat.modules: - module: netflow log: enabled: true var: netflow_host: 0.0.0.0 netflow_port: 2055 ``` --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 7 ++ filebeat/docs/modules/netflow.asciidoc | 80 +++++++++++++++++++ filebeat/docs/modules_list.asciidoc | 2 + x-pack/filebeat/filebeat.reference.yml | 8 ++ x-pack/filebeat/include/list.go | 1 + .../filebeat/module/netflow/_meta/config.yml | 6 ++ .../module/netflow/_meta/docs.asciidoc | 67 ++++++++++++++++ .../filebeat/module/netflow/_meta/fields.yml | 6 ++ x-pack/filebeat/module/netflow/fields.go | 23 ++++++ .../module/netflow/log/config/netflow.yml | 6 ++ .../module/netflow/log/ingest/pipeline.json | 29 +++++++ .../filebeat/module/netflow/log/manifest.yml | 20 +++++ .../filebeat/modules.d/netflow.yml.disabled | 9 +++ 14 files changed, 265 insertions(+) create mode 100644 filebeat/docs/modules/netflow.asciidoc create mode 100644 x-pack/filebeat/module/netflow/_meta/config.yml create mode 100644 x-pack/filebeat/module/netflow/_meta/docs.asciidoc create mode 100644 x-pack/filebeat/module/netflow/_meta/fields.yml create mode 100644 x-pack/filebeat/module/netflow/fields.go create mode 100644 x-pack/filebeat/module/netflow/log/config/netflow.yml create mode 100644 x-pack/filebeat/module/netflow/log/ingest/pipeline.json create mode 100644 x-pack/filebeat/module/netflow/log/manifest.yml create mode 100644 x-pack/filebeat/modules.d/netflow.yml.disabled diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 321e83d31f6..77906646291 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -243,6 +243,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Populate more ECS fields in the Suricata module. {pull}10006[10006] - Add ISO8601 timestamp support in syslog metricset. {issue}8716[8716] {pull}10736[10736] - Add more info to message logged when a duplicated symlink file is found {pull}10845[10845] +- Add Netflow module to enrich flow events with geoip data. {pull}10877[10877] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 6df8c561056..d1fe115e4ae 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -33,6 +33,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -10621,6 +10622,12 @@ type: short -- +[[exported-fields-netflow-module]] +== NetFlow fields + +Module for receiving NetFlow and IPFIX flow records over UDP. The module does not add fields beyond what the netflow input provides. + + [[exported-fields-nginx]] == Nginx fields diff --git a/filebeat/docs/modules/netflow.asciidoc b/filebeat/docs/modules/netflow.asciidoc new file mode 100644 index 00000000000..f3f9c8c0191 --- /dev/null +++ b/filebeat/docs/modules/netflow.asciidoc @@ -0,0 +1,80 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-netflow]] +[role="xpack"] + +:modulename: netflow +:has-dashboards: false + +== NetFlow module + +This is a module for receiving NetFlow and IPFIX flow records over UDP. This +input supports NetFlow versions 1, 5, 6, 7, 8 and 9, as well as IPFIX. For +NetFlow versions older than 9, fields are mapped automatically to NetFlow v9. + +This module wraps the <> to enrich the +flow records with geolocation information about the IP endpoints by using +Elasticsearch Ingest Node. + +[float] +=== Compatibility + +This module requires the {elasticsearch-plugins}/ingest-geoip.html[ingest-geoip] +Elasticsearch plugins. + +include::../include/running-modules.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +The fileset is by default configured to listen for UDP traffic on +`localhost:2055`. For most uses cases you will want to set the `netflow_host` +variable to allow the input bind to all interfaces so that it can receive +traffic from network devices. + +["source","yaml",subs="attributes"] +----- +- module: netflow + log: + enabled: true + var: + netflow_host: 0.0.0.0 + netflow_port: 2055 +----- + +`var.netflow_host`:: Address to find to. Defaults to `localhost`. + +`var.netflow_port`:: Port to listen on. Defaults to `2055`. + +`var.max_message_size`:: The maximum size of the message received over UDP. +The default is `10KiB`. + +`var.expiration_timeout`:: The time before an idle session or unused template is +expired. Only applicable to v9 and IPFIX protocols. A value of zero disables +expiration. + +`var.queue_size`:: The maximum number of packets that can be queued for +processing. Use this setting to avoid packet-loss when dealing with occasional +bursts of traffic. + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 1324bcc4996..ea3b3d74997 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -16,6 +16,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -43,6 +44,7 @@ include::modules/logstash.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] include::modules/nats.asciidoc[] +include::modules/netflow.asciidoc[] include::modules/nginx.asciidoc[] include::modules/osquery.asciidoc[] include::modules/postgresql.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 8d41832ce47..1d0217307c2 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -312,6 +312,14 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: +#------------------------------- NetFlow Module ------------------------------- +- module: netflow + log: + enabled: true + var: + netflow_host: localhost + netflow_port: 2055 + #-------------------------------- Nginx Module -------------------------------- #- module: nginx # Access logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index f4550a6e898..c6332bde38f 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -10,6 +10,7 @@ import ( // Import packages that need to register themselves. _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/iptables" + _ "github.com/elastic/beats/x-pack/filebeat/module/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" _ "github.com/elastic/beats/x-pack/filebeat/module/zeek" ) diff --git a/x-pack/filebeat/module/netflow/_meta/config.yml b/x-pack/filebeat/module/netflow/_meta/config.yml new file mode 100644 index 00000000000..20d1905b6f4 --- /dev/null +++ b/x-pack/filebeat/module/netflow/_meta/config.yml @@ -0,0 +1,6 @@ +- module: netflow + log: + enabled: true + var: + netflow_host: localhost + netflow_port: 2055 diff --git a/x-pack/filebeat/module/netflow/_meta/docs.asciidoc b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc new file mode 100644 index 00000000000..334a219ba84 --- /dev/null +++ b/x-pack/filebeat/module/netflow/_meta/docs.asciidoc @@ -0,0 +1,67 @@ +[role="xpack"] + +:modulename: netflow +:has-dashboards: false + +== NetFlow module + +This is a module for receiving NetFlow and IPFIX flow records over UDP. This +input supports NetFlow versions 1, 5, 6, 7, 8 and 9, as well as IPFIX. For +NetFlow versions older than 9, fields are mapped automatically to NetFlow v9. + +This module wraps the <> to enrich the +flow records with geolocation information about the IP endpoints by using +Elasticsearch Ingest Node. + +[float] +=== Compatibility + +This module requires the {elasticsearch-plugins}/ingest-geoip.html[ingest-geoip] +Elasticsearch plugins. + +include::../include/running-modules.asciidoc[] + +include::../include/configuring-intro.asciidoc[] + +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +The fileset is by default configured to listen for UDP traffic on +`localhost:2055`. For most uses cases you will want to set the `netflow_host` +variable to allow the input bind to all interfaces so that it can receive +traffic from network devices. + +["source","yaml",subs="attributes"] +----- +- module: netflow + log: + enabled: true + var: + netflow_host: 0.0.0.0 + netflow_port: 2055 +----- + +`var.netflow_host`:: Address to find to. Defaults to `localhost`. + +`var.netflow_port`:: Port to listen on. Defaults to `2055`. + +`var.max_message_size`:: The maximum size of the message received over UDP. +The default is `10KiB`. + +`var.expiration_timeout`:: The time before an idle session or unused template is +expired. Only applicable to v9 and IPFIX protocols. A value of zero disables +expiration. + +`var.queue_size`:: The maximum number of packets that can be queued for +processing. Use this setting to avoid packet-loss when dealing with occasional +bursts of traffic. + +:has-dashboards!: + +:fileset_ex!: + +:modulename!: diff --git a/x-pack/filebeat/module/netflow/_meta/fields.yml b/x-pack/filebeat/module/netflow/_meta/fields.yml new file mode 100644 index 00000000000..fc4bf3bb887 --- /dev/null +++ b/x-pack/filebeat/module/netflow/_meta/fields.yml @@ -0,0 +1,6 @@ +- key: netflow-module + title: NetFlow + description: > + Module for receiving NetFlow and IPFIX flow records over UDP. The module + does not add fields beyond what the netflow input provides. + fields: diff --git a/x-pack/filebeat/module/netflow/fields.go b/x-pack/filebeat/module/netflow/fields.go new file mode 100644 index 00000000000..d71d8ceb53f --- /dev/null +++ b/x-pack/filebeat/module/netflow/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package netflow + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "netflow", asset.ModuleFieldsPri, AssetNetflow); err != nil { + panic(err) + } +} + +// AssetNetflow returns asset data. +// This is the base64 encoded gzipped contents of module/netflow. +func AssetNetflow() string { + return "eJw8jjFOw0AQRfs9xbtAcoAtqFCkFKAUINGazBiPWHas3Ymt3B4Z4fTv/f8OfOs9UzXG4uvhx+VWNEFYFM28apyKrwlE+7XZHOY185QAXv5gRm80vaotVr92g6EK58vp/ME2vAHepOOLNt6fL0feJuVxB+LaqR4MIoymRTqfevcqrNMQxKR7JVbnWzA3X0y0HxP/Qk6/AQAA//9CcUYh" +} diff --git a/x-pack/filebeat/module/netflow/log/config/netflow.yml b/x-pack/filebeat/module/netflow/log/config/netflow.yml new file mode 100644 index 00000000000..b6d045353db --- /dev/null +++ b/x-pack/filebeat/module/netflow/log/config/netflow.yml @@ -0,0 +1,6 @@ +type: netflow +protocols: [v1, v5, v6, v7, v8, v9, ipfix] +host: '{{.netflow_host}}:{{.netflow_port}}' +max_message_size: '{{.max_message_size}}' +expiration_timeout: '{{.expiration_timeout}}' +queue_size: {{.queue_size}} diff --git a/x-pack/filebeat/module/netflow/log/ingest/pipeline.json b/x-pack/filebeat/module/netflow/log/ingest/pipeline.json new file mode 100644 index 00000000000..6eeddb6f430 --- /dev/null +++ b/x-pack/filebeat/module/netflow/log/ingest/pipeline.json @@ -0,0 +1,29 @@ +{ + "description": "Pipeline for Filebeat NetFlow", + "processors": [ + { + "geoip": { + "if": "ctx.source?.geo == null", + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "if": "ctx.destination?.geo == null", + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + } + ], + "on_failure": [ + { + "set": { + "field": "error.message", + "value": "{{ _ingest.on_failure_message }}" + } + } + ] +} diff --git a/x-pack/filebeat/module/netflow/log/manifest.yml b/x-pack/filebeat/module/netflow/log/manifest.yml new file mode 100644 index 00000000000..4e8ee131736 --- /dev/null +++ b/x-pack/filebeat/module/netflow/log/manifest.yml @@ -0,0 +1,20 @@ +module_version: "1.0" + +var: + - name: netflow_host + default: localhost + - name: netflow_port + default: 2055 + - name: max_message_size + default: 10KiB + - name: expiration_timeout + default: 30m + - name: queue_size + default: 8192 + +ingest_pipeline: ingest/pipeline.json +input: config/netflow.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/modules.d/netflow.yml.disabled b/x-pack/filebeat/modules.d/netflow.yml.disabled new file mode 100644 index 00000000000..522307d7e71 --- /dev/null +++ b/x-pack/filebeat/modules.d/netflow.yml.disabled @@ -0,0 +1,9 @@ +# Module: netflow +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-netflow.html + +- module: netflow + log: + enabled: true + var: + netflow_host: localhost + netflow_port: 2055