From 120f9319cd28f54cdbd7e636a49d467645a3865e Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Mon, 2 Nov 2020 21:40:03 +0100 Subject: [PATCH] [Filebeat][Fortinet] Fixing kv split for when assign-ip is not an IP and for date checking when eventtime is missing (#22361) * fixing assignip and making sure it does not error on missing eventtime field * updating changelog --- CHANGELOG.next.asciidoc | 1 + .../fortinet/firewall/ingest/pipeline.yml | 31 ++++++---- .../fortinet/firewall/test/fortinet.log | 1 + .../firewall/test/fortinet.log-expected.json | 61 +++++++++++++++++++ 4 files changed, 81 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 271b46b1133..e746f533cc8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -290,6 +290,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382] - Fix syslog RFC 5424 parsing in the CheckPoint module. {pull}21854[21854] - Fix incorrect connection state mapping in zeek connection pipeline. {pull}22151[22151] {issue}22149[22149] +- Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] *Heartbeat* diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index eeb5368db55..b53373ec030 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -11,10 +11,18 @@ processors: field: syslog5424_sd field_split: " (?=[a-z\\_\\-]+=)" value_split: "=" - prefix: "fortinet.firewall." + prefix: "fortinet.tmp." ignore_missing: true ignore_failure: false trim_value: "\"" +- remove: + field: fortinet.tmp.assignip + if: "ctx.fortinet?.tmp?.assignip == 'N/A'" + ignore_missing: true +- rename: + field: fortinet.tmp + target_field: fortinet.firewall + ignore_missing: true - set: field: observer.vendor value: Fortinet @@ -65,41 +73,41 @@ processors: field: fortinet.firewall.eventtime pattern: "\\d{6}$" replacement: "" - if: "(ctx.fortinet?.firewall?.eventtime).length() > 18" + if: "ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 18" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX timezone: "{{fortinet.firewall.tz}}" - if: "ctx.fortinet?.firewall?.tz != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" + if: "ctx.fortinet?.firewall?.tz != null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX_MS - if: "ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() > 11" + if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() > 11" - date: field: fortinet.firewall.eventtime target_field: event.start formats: - UNIX - if: "ctx.fortinet?.firewall?.tz == null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" -- rename: - field: fortinet.firewall.devname - target_field: observer.name - ignore_missing: true + if: "ctx.fortinet?.firewall?.tz == null && ctx.fortinet?.firewall?.eventtime != null && (ctx.fortinet?.firewall?.eventtime).length() <= 11" - script: lang: painless source: "ctx.event.duration = Long.parseLong(ctx.fortinet.firewall.duration) * 1000000000" if: "ctx.fortinet?.firewall?.duration != null" +- rename: + field: fortinet.firewall.devname + target_field: observer.name + ignore_missing: true - rename: field: fortinet.firewall.devid target_field: observer.serial_number @@ -126,9 +134,6 @@ processors: field: fortinet.firewall.level target_field: log.level ignore_missing: true -- remove: - field: fortinet.firewall.assignip - if: "ctx.fortinet?.firewall?.assignip == 'N/A'" - remove: field: fortinet.firewall.dstip if: "ctx.fortinet?.firewall?.dstip == 'N/A'" diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log index 410daa4405b..e3c4ddd0d9f 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log @@ -29,3 +29,4 @@ <188>date=2020-04-23 time=12:14:39 devname="firewall3" devid="oldfwid" logid="0000000011" type="traffic" subtype="forward" level="warning" vd="root" eventtime=1587230079841464445 tz="-0500" srcip=192.168.1.1 srcport=62493 srcintf="port1" srcintfrole="lan" dstip=192.168.100.100 dstport=1235 dstintf="newinterface" dstintfrole="undefined" sessionid=54234 proto=17 action="ip-conn" policyid=49 policytype="policy" poluuid="654cc-b6542-53467u8-e45234-1566casd35f7836" policyname="oldpolicyname" user="elasticsuper" authserver="FSSO_newfsso" service="udp/12302" dstcountry="Reserved" srccountry="Reserved" appcat="unscanned" crscore=5 craction=63332144 crlevel="low" <189>date=2020-04-23 time=12:14:28 devname="firewall3" devid="oldfwid" logid="0000000013" type="traffic" subtype="forward" level="notice" vd="root" eventtime=1587230069291463928 tz="-0500" srcip=192.168.50.50 srcport=56603 srcintf="port1" srcintfrole="lan" dstip=8.8.8.8 dstport=442 dstintf="wan1" dstintfrole="wan" sessionid=2345 proto=6 action="close" policyid=2365 policytype="policy" poluuid="654644c-b064-fdgdf3425-f003-1234ghdf682e05f" policyname="someoldpolicyname" user="elasticuser" group="testgroup" authserver="FSSO_something" service="HTTPS" dstcountry="Netherlands" srccountry="Reserved" trandisp="snat" transip=23.23.23.23 transport=603 appid=43540 app="Skype.Portals" appcat="Collaboration" apprisk="elevated" applist="someapplist" appact="detected" duration=126 sentbyte=923 rcvdbyte=77654 sentpkt=113 rcvdpkt=70 vwlid=4 vwlquality="Seq_num(3), alive, selected" wanin=1130 wanout=6671 lanin=1406 lanout=146506 utmaction="block" countweb=1 countapp=1 crscore=5 craction=6144 crlevel="low" <190>date=2019-05-15 time=18:03:36 logid="1059028704" type="utm" subtype="app-ctrl" eventtype="app-ctrl-all" level="information" vd="root" eventtime=1557968615 appid=40568 srcip=10.1.100.22 dstip=195.8.215.136 srcport=50798 dstport=443 srcintf="port10" srcintfrole="lan" dstintf="port9" dstintfrole="wan" proto=6 service="HTTPS" direction="outgoing" policyid=1 sessionid=4414 applist="block-social.media" appcat="Web.Client" app="HTTPS.BROWSER" action="pass" hostname="www.dailymotion.com" incidentserialno=1962906680 url="/" msg="Web.Client: HTTPS.BROWSER," apprisk="medium" scertcname="*.dailymotion.com" scertissuer="DigiCert SHA2 High Assurance Server CA" +<190>date=2020-11-02 time=08:11:38 devname=testfirewall devid=newrouterid logid=0101037127 type="event" subtype=vpn level=notice vd=root logdesc="Progress IPsec phase 1" msg="progress IPsec phase 1" action=negotiate remip=8.8.8.8 locip=10.10.10.10 remport=500 locport=500 outintf="port1" cookies="125cbf9ee8349965/0000000000000000" user="N/A" group="N/A" xauthuser="N/A" xauthgroup="N/A" assignip=N/A vpntunnel="P1_Test" status=success init=local mode=aggressive dir=outbound stage=1 role=initiator result=OK diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 1bc7032f6d2..090b6db4e3f 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -1968,5 +1968,66 @@ "tls.server.x509.subject.common_name": "*.dailymotion.com", "url.domain": "www.dailymotion.com", "url.path": "/" + }, + { + "@timestamp": "2020-11-02T08:11:38.000Z", + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.port": 500, + "event.category": [ + "network" + ], + "event.code": "0101037127", + "event.dataset": "fortinet.firewall", + "event.kind": "event", + "event.module": "fortinet", + "event.outcome": "success", + "event.type": [ + "connection" + ], + "fileset.name": "firewall", + "fortinet.firewall.action": "negotiate", + "fortinet.firewall.cookies": "125cbf9ee8349965/0000000000000000", + "fortinet.firewall.init": "local", + "fortinet.firewall.mode": "aggressive", + "fortinet.firewall.outintf": "port1", + "fortinet.firewall.result": "OK", + "fortinet.firewall.role": "initiator", + "fortinet.firewall.stage": "1", + "fortinet.firewall.status": "success", + "fortinet.firewall.subtype": "vpn", + "fortinet.firewall.type": "event", + "fortinet.firewall.vd": "root", + "fortinet.firewall.vpntunnel": "P1_Test", + "fortinet.firewall.xauthgroup": "N/A", + "fortinet.firewall.xauthuser": "N/A", + "input.type": "log", + "log.level": "notice", + "log.offset": 17123, + "message": "progress IPsec phase 1", + "network.direction": "outbound", + "observer.name": "testfirewall", + "observer.product": "Fortigate", + "observer.serial_number": "newrouterid", + "observer.type": "firewall", + "observer.vendor": "Fortinet", + "related.ip": [ + "10.10.10.10", + "8.8.8.8" + ], + "rule.description": "Progress IPsec phase 1", + "service.type": "fortinet", + "source.ip": "10.10.10.10", + "source.port": 500, + "tags": [ + "fortinet-firewall", + "forwarded" + ] } ] \ No newline at end of file