From 2fe9e2ae711d0e87874531258e53c970a8b791ca Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 5 Nov 2020 14:52:31 -0500 Subject: [PATCH] [Filebeat] Refactor Suricata pipeline to use Ingest Node more (#22291) Refactor the Suricata module pipeline to use Ingest Node more. I moved most processing from Beat processors to Ingest Node. I created a DNS and TLS pipeline to house all of the processing for those protocols respectively. Other changes - Added DNS response IPs to `related.ip`. - Some DNS events were missing `dns.header_flags`. --- CHANGELOG.next.asciidoc | 1 + .../module/suricata/eve/config/eve.yml | 365 +----------------- .../suricata/eve/ingest/dns-answer-v1.yml | 39 ++ .../suricata/eve/ingest/dns-answer-v2.yml | 42 ++ .../module/suricata/eve/ingest/dns.yml | 93 +++++ .../module/suricata/eve/ingest/pipeline.yml | 270 +++++++------ .../module/suricata/eve/ingest/tls.yml | 188 +++++++++ .../filebeat/module/suricata/eve/manifest.yml | 7 +- .../eve/test/eve-dns-4.1.4.log-expected.json | 90 +++++ .../eve/test/eve-small.log-expected.json | 7 + 10 files changed, 629 insertions(+), 473 deletions(-) create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/dns.yml create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/tls.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d60b41bb9f2..cc71ca69583 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -656,6 +656,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add max_number_of_messages config into s3 input. {pull}21993[21993] - Update Okta documentation for new stateful restarts. {pull}22091[22091] - Copy tag names from MISP data into events. {pull}21664[21664] +- Added DNS response IP addresses to `related.ip` in Suricata module. {pull}22291[22291] - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] - Added `event.ingested` field to data from the Netflow module. {pull}22412[22412] diff --git a/x-pack/filebeat/module/suricata/eve/config/eve.yml b/x-pack/filebeat/module/suricata/eve/config/eve.yml index 879bcdd8e35..231127914c2 100644 --- a/x-pack/filebeat/module/suricata/eve/config/eve.yml +++ b/x-pack/filebeat/module/suricata/eve/config/eve.yml @@ -42,363 +42,16 @@ processors: {{ if .community_id }} - community_id: {{ end }} - - if: - equals: - suricata.eve.event_type: dns - then: - - convert: - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.id, to: dns.id, type: string} - - {from: suricata.eve.dns.rcode, to: dns.response_code} - - {from: suricata.eve.dns.type, to: dns.type} - - convert: - when.equals.dns.type: query - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.rrname, to: dns.question.name} - - {from: suricata.eve.dns.rrtype, to: dns.question.type} - - if: - and: - - equals.dns.type: answer - - equals.suricata.eve.dns.version: 2 - then: - - convert: - ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.dns.rrname, to: dns.question.name} - - {from: suricata.eve.dns.rrtype, to: dns.question.type} - - registered_domain: - ignore_missing: true - ignore_failure: true - field: dns.question.name - target_field: dns.question.registered_domain - - script: - id: eve_process - lang: javascript - source: >- - function addEcsCategorization(evt) { - var event_type = evt.Get("suricata.eve.event_type"); - if (event_type == null) { - return; - } - var catArray = []; - var typeArray = []; - evt.Put("suricata.eve.event_type", event_type.toLowerCase()); - switch (event_type.toLowerCase()) { - case "alert": - evt.Put("event.kind", "alert"); - catArray.push("network"); - catArray.push("intrusion_detection"); - break; - case "anomaly": - evt.Put("event.kind", "event"); - catArray.push("network"); - break; - case "http": - evt.Put("event.kind", "event"); - catArray.push("network"); - catArray.push("web"); - typeArray.push("access"); - typeArray.push("protocol"); - evt.Put("network.protocol", "http"); - var status = evt.Get("suricata.eve.http.status"); - if (status == null) { - break; - } - if (status < 400) { - evt.Put("event.outcome", "success"); - } - if (status >= 400 ) { - evt.Put("event.outcome", "failure"); - } - break; - case "dns": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "dns"); - break; - case "ftp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ftp"); - break; - case "ftp_data": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ftp"); - break; - case "tls": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "tls"); - break; - case "tftp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "tftp"); - break; - case "smb": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "smb"); - break; - case "ssh": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "ssh"); - break; - case "flow": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("connection"); - var state = evt.Get("suricata.eve.flow.state"); - if (state == null) { - break; - } - switch (state) { - case "new": - typeArray.push("start"); - break; - case "closed": - typeArray.push("end"); - break; - } - break; - case "rdp": - evt.Put("event.kind", "event"); - catArray.push("network"); - typeArray.push("protocol"); - evt.Put("network.protocol", "rdp"); - break; - case "stats": - evt.Put("event.kind", "metric"); - break; - default: - evt.Put("event.kind", "event"); - catArray.push("network"); - } - if (catArray.length > 0) { - evt.Put("event.category", catArray); - } - if (typeArray.length > 0) { - evt.Put("event.type", typeArray); - } - } - function setDnsV1Answers(evt) { - var dns_type = evt.Get("dns.type") - if (dns_type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version == "2") { - return; - } - var name = evt.Get("suricata.eve.dns.rrname"); - var data = evt.Get("suricata.eve.dns.rdata"); - var type = evt.Get("suricata.eve.dns.rrtype"); - var ttl = evt.Get("suricata.eve.dns.ttl"); - var answer = {}; - if (name) { - answer.name = name; - } - if (data) { - answer.data = data; - } - if (type) { - answer.type = type; - } - if (ttl) { - answer.ttl = ttl; - } - if (Object.keys(answer).length === 0) { - return; - } - evt.Put("dns.answers", [answer]); - } - function addDnsV2Answers(evt) { - var type = evt.Get("dns.type") - if (type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version != 2) { - return; - } - var answers = evt.Get("suricata.eve.dns.answers"); - if (!answers) { - return; - } - evt.Delete("suricata.eve.dns.answers"); - var resolvedIps = []; - for (var i = 0; i < answers.length; i++) { - var answer = answers[i]; - - // Rename properties. - var name = answer["rrname"]; - delete answer["rrname"]; - var type = answer["rrtype"]; - delete answer["rrtype"]; - var data = answer["rdata"]; - delete answer["rdata"]; - - answer["name"] = name; - answer["type"] = type; - answer["data"] = data; - - // Append IP addresses to dns.resolved_ip. - if (type === "A" || type === "AAAA") { - resolvedIps.push(data); - } - } - evt.Put("dns.answers", answers); - if (resolvedIps.length > 0) { - evt.Put("dns.resolved_ip", resolvedIps); - } - } - function addDnsV2HeaderFlags(evt) { - var type = evt.Get("dns.type") - if (type != "answer") { - return; - } - var version = evt.Get("suricata.eve.dns.version") - if (version != 2) { - return; - } - var flag = evt.Get("suricata.eve.dns.aa"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "AA"); - } - - flag = evt.Get("suricata.eve.dns.tc"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "TC"); - } - - flag = evt.Get("suricata.eve.dns.rd"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "RD"); - } - - flag = evt.Get("suricata.eve.dns.ra"); - if (flag === true) { - evt.AppendTo("dns.header_flags", "RA"); - } - } - function addTopLevelDomain(evt) { - var rd = evt.Get("dns.question.registered_domain"); - if (rd == null) { - return; - } - var firstPeriod = rd.indexOf("."); - if (firstPeriod == -1) { - return; - } - evt.Put("dns.question.top_level_domain", rd.substr(firstPeriod + 1)); - } - function cleanupAppProto(evt) { - var proto = evt.Get("suricata.eve.app_proto"); - if (proto == null){ - return; - } - switch (proto.toLowerCase()) { - case "failed": - case "template": - case "template-rust": - break; - case "ftp-data": - evt.Put("network.protocol", "ftp"); - break; - default: - evt.Put("network.protocol", proto.toLowerCase()); - } - evt.Delete("suricata.eve.app_proto"); - } - function addRelatedIps(evt) { - var src_ip = evt.Get("source.ip"); - if (src_ip != null) { - evt.AppendTo("related.ip", src_ip); - } - var dst_ip = evt.Get("destination.ip"); - if (dst_ip != null) { - evt.AppendTo("related.ip", dst_ip); - } - } - function addTlsVersion(evt) { - var tls_version = evt.Get("suricata.eve.tls.version"); - if (tls_version == null) { - return; - } - var parts = tls_version.split(" "); - if (parts.length < 2) { - return; - } - evt.Put("tls.version_protocol", parts[0].toLowerCase()); - evt.Put("tls.version", parts[1]); - } - function cleanupTlsSni(evt) { - var sni = evt.Get("suricata.eve.tls.sni"); - if (sni == null) { - return; - } - if ("." == sni.charAt(sni.length - 1)) { - evt.Put("suricata.eve.tls.sni", sni.substring(0, sni.length - 1)); - } - } - function process(evt) { - var event_type = evt.Get("suricata.eve.event_type"); - - addEcsCategorization(evt); - if (event_type == "dns") { - setDnsV1Answers(evt); - addDnsV2Answers(evt); - addDnsV2HeaderFlags(evt); - addTopLevelDomain(evt); - } - cleanupAppProto(evt); - addRelatedIps(evt); - addTlsVersion(evt); - cleanupTlsSni(evt); - } - - convert: + - registered_domain: + when: + or: + - equals.suricata.eve.dns.type: query + # V2 events always include the query data. + - equals.suricata.eve.dns.version: 2 ignore_missing: true - fail_on_error: false - mode: copy - fields: - - {from: suricata.eve.tls.subject, to: tls.server.subject} - - {from: suricata.eve.tls.issuerdn, to: tls.server.issuer} - - {from: suricata.eve.tls.session_resumed, to: tls.resumed, type: boolean} - - {from: suricata.eve.tls.fingerprint, to: tls.server.hash.sha1} - - {from: suricata.eve.tls.sni, to: tls.client.server_name} - - {from: suricata.eve.tls.sni, to: destination.domain} - - {from: suricata.eve.tls.ja3s.hash, to: tls.server.ja3s} - - {from: suricata.eve.tls.ja3.hash, to: tls.client.ja3} - - {from: suricata.eve.tls.certificate, to: tls.server.certificate} - - {from: suricata.eve.tls.chain, to: tls.server.certificate_chain} - - drop_fields: - ignore_missing: true - fields: - - suricata.eve.dns.aa - - suricata.eve.dns.tc - - suricata.eve.dns.rd - - suricata.eve.dns.ra - - suricata.eve.dns.qr - - suricata.eve.dns.version - - suricata.eve.dns.flags - - suricata.eve.dns.grouped + ignore_failure: true + field: suricata.eve.dns.rrname + target_field: dns.question.registered_domain - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml new file mode 100644 index 00000000000..e915537365d --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v1.yml @@ -0,0 +1,39 @@ +--- +description: Pipeline for Suricata DNS answers v1 + +# Suricata DNS v1 events contain a single answer. Multiple events are created +# to represent all of the answers. +processors: + - script: + lang: painless + tag: suricata_dns_answer_v1 + source: | + def name = ctx?.suricata?.eve?.dns?.rrname; + def data = ctx?.suricata?.eve?.dns?.rdata; + def type = ctx?.suricata?.eve?.dns?.rrtype; + def ttl = ctx?.suricata?.eve?.dns?.ttl; + + def answer = [:]; + if (name != null) { + answer["name"] = name; + } + if (data != null) { + answer["data"] = data; + } + if (type != null) { + answer["type"] = type; + } + if (ttl != null) { + answer["ttl"] = ttl; + } + if (!answer.isEmpty()) { + ctx.dns.answers = [answer]; + } + + if (type == "A" || type == "AAAA") { + ctx.dns.resolved_ip = [data]; + } +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml new file mode 100644 index 00000000000..a9e77c28549 --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns-answer-v2.yml @@ -0,0 +1,42 @@ +--- +description: Pipeline for Suricata DNS answers v2 + +# Suricata DNS v2 events contain all answers in a single event. +processors: + - rename: + field: suricata.eve.dns.answers + target_field: dns.answers + ignore_missing: true + - script: + if: ctx?.dns?.answers != null + lang: painless + tag: suricata_dns_answers_v2 + source: | + def resolvedIps = new ArrayList(); + for (def answer : ctx?.dns?.answers) { + // Normalize field names to match ECS. + def name = answer.remove("rrname"); + if (name != null) { + answer["name"] = name; + } + def type = answer.remove("rrtype"); + if (type != null) { + answer["type"] = type; + } + def data = answer.remove("rdata"); + if (data != null) { + answer["data"] = data; + } + + if (type == "A" || type == "AAAA") { + resolvedIps.add(data); + } + } + + if (resolvedIps.size() > 0) { + ctx.dns.resolved_ip = resolvedIps; + } +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/suricata/eve/ingest/dns.yml b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml new file mode 100644 index 00000000000..f2c1127ddee --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/dns.yml @@ -0,0 +1,93 @@ +--- +description: Pipeline for Suricata DNS Events + +processors: + - set: + field: dns.id + value: '{{suricata.eve.dns.id}}' + ignore_empty_value: true + - set: + field: dns.response_code + value: '{{suricata.eve.dns.rcode}}' + ignore_empty_value: true + - set: + field: dns.type + value: '{{suricata.eve.dns.type}}' + ignore_empty_value: true + - set: + # V2 events always include the query data. + if: >- + ctx?.dns?.type == "query" || + ctx?.suricata?.eve?.dns?.version == 2 + field: dns.question.name + value: '{{suricata.eve.dns.rrname}}' + ignore_empty_value: true + - set: + # V2 events always include the query data. + if: >- + ctx?.dns?.type == "query" || + ctx?.suricata?.eve?.dns?.version == 2 + field: dns.question.type + value: '{{suricata.eve.dns.rrtype}}' + ignore_empty_value: true + - pipeline: + if: >- + ctx?.dns?.type == "answer" && + ctx?.suricata?.eve?.dns?.version == null + name: '{< IngestPipeline "dns-answer-v1" >}' + - pipeline: + if: >- + ctx?.dns?.type == "answer" && + ctx?.suricata?.eve?.dns?.version == 2 + name: '{< IngestPipeline "dns-answer-v2" >}' + - foreach: + field: dns.resolved_ip + ignore_missing: true + processor: + append: + field: related.ip + value: + - '{{_ingest._value}}' + allow_duplicates: false + - script: + if: ctx?.dns?.question?.registered_domain != null + tag: suricata_dns_top_level_domain + lang: painless + source: | + def rd = ctx.dns.question.registered_domain; + def firstDot = rd.indexOf("."); + if (firstDot == -1) { + return; + } + ctx.dns.question.top_level_domain = rd.substring(firstDot + 1); + - append: + if: ctx?.suricata?.eve?.dns?.aa == true + field: dns.header_flags + value: AA + - append: + if: ctx?.suricata?.eve?.dns?.tc == true + field: dns.header_flags + value: TC + - append: + if: ctx?.suricata?.eve?.dns?.rd == true + field: dns.header_flags + value: RD + - append: + if: ctx?.suricata?.eve?.dns?.ra == true + field: dns.header_flags + value: RA + - remove: + field: + - suricata.eve.dns.aa + - suricata.eve.dns.tc + - suricata.eve.dns.rd + - suricata.eve.dns.ra + - suricata.eve.dns.qr + - suricata.eve.dns.version + - suricata.eve.dns.flags + - suricata.eve.dns.grouped + ignore_missing: true +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index e132a8acdde..206f6b06e46 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -5,6 +5,148 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' + + # Handle the different Suricata event types. + - lowercase: + field: suricata.eve.event_type + ignore_missing: true + - script: + lang: painless + ignore_failure: true + params: + alert: + kind: alert + category: + - network + - intrusion_detection + dns: + type: + - protocol + network_protocol: dns + flow: + type: + - connection + ftp: + type: + - protocol + network_protocol: ftp + ftp_data: + type: + - protocol + network_protocol: ftp + http: + category: + - network + - web + type: + - access + - protocol + network_protocol: http + http2: + category: + - network + - web + type: + - access + - protocol + network_protocol: http + mqtt: + type: + - protocol + network_protocol: mqtt + smb: + type: + - protocol + network_protocol: smb + ssh: + type: + - protocol + network_protocol: ssh + stats: + kind: metric + tftp: + type: + - protocol + network_protocol: tftp + tls: + type: + - protocol + network_protocol: tls + rdp: + type: + - protocol + network_protocol: rdp + rfb: # RFB (Remote Framebuffer Protocol) + type: + - protocol + network_protocol: rdp + + source: | + ctx.event.kind = 'event'; + ctx.event.category = ['network']; + def type_params = params.get(ctx?.suricata?.eve?.event_type); + if (type_params == null) { + return; + } + type_params.forEach((k, v) -> { + if ('network_protocol' == k) { + if (ctx.network == null) { + ctx.network = ['protocol': v]; + } else { + ctx.network.protocol = v; + } + } else { + ctx.event[k] = v; + } + }); + + ## Anomaly and Alert + - lowercase: + field: suricata.eve.app_proto + ignore_missing: true + - set: + if: ctx?.suricata?.eve?.app_proto == "ftp-data" + field: network.protocol + value: ftp + - set: + if: >- + ctx?.suricata?.eve?.app_proto != "failed" && + ctx?.suricata?.eve?.app_proto != "template" && + ctx?.suricata?.eve?.app_proto != "template-rust" + field: network.protocol + value: '{{suricata.eve.app_proto}}' + ignore_empty_value: true + ## HTTP + - set: + if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status < 400 + field: event.outcome + value: success + - set: + if: ctx?.suricata?.eve?.event_type == "http" && ctx?.suricata?.eve?.http?.status >= 400 + field: event.outcome + value: failure + ## DNS + - pipeline: + if: >- + ctx?.network?.protocol == "dns" + name: '{< IngestPipeline "dns" >}' + ## TLS + - pipeline: + if: ctx?.network?.protocol == "tls" + name: '{< IngestPipeline "tls" >}' + ## Flow + - append: + if: ctx?.suricata?.eve?.flow?.state == "new" + field: event.type + value: + - start + - append: + if: ctx?.suricata?.eve?.flow?.state == "closed" + field: event.type + value: + - end + + - set: value: "{{suricata.eve.http.http_method}}" field: http.request.method @@ -17,11 +159,13 @@ processors: if: ctx.suricata?.eve?.http?.hostname != null value: '{{suricata.eve.http.hostname}}' field: destination.domain + allow_duplicates: false - remove: field: suricata.eve.http.hostname ignore_failure: true - script: lang: painless + tag: suricata_deduplicate_dest_domain source: > def domain = ctx.destination?.domain; if (domain instanceof Collection) { @@ -219,125 +363,21 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - uppercase: - field: tls.server.hash.sha1 - ignore_missing: true - - split: - field: tls.server.hash.sha1 - separator: ":" - ignore_missing: true - - join: - field: tls.server.hash.sha1 - separator: "" - ignore_failure: true - - append: - field: related.hash - value: "{{tls.server.hash.sha1}}" - if: "ctx?.tls?.server?.hash?.sha1 != null" - - gsub: - field: suricata.eve.tls.issuerdn - pattern: \\, - replacement: "" - ignore_missing: true - - kv: - field: suricata.eve.tls.issuerdn - field_split: ', ' - value_split: '=' - target_field: suricata.eve.tls.kv_issuerdn - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.C - target_field: tls.server.x509.issuer.country - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.CN - target_field: tls.server.x509.issuer.common_name - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.L - target_field: tls.server.x509.issuer.locality - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.O - target_field: tls.server.x509.issuer.organization - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.OU - target_field: tls.server.x509.issuer.organizational_unit - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_issuerdn.ST - target_field: tls.server.x509.issuer.state_or_province - ignore_missing: true - - gsub: - field: suricata.eve.tls.subject - pattern: \\, - replacement: "" - ignore_missing: true - - kv: - field: suricata.eve.tls.subject - field_split: ', ' - value_split: '=' - target_field: suricata.eve.tls.kv_subject - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.C - target_field: tls.server.x509.subject.country - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.CN - target_field: tls.server.x509.subject.common_name - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.L - target_field: tls.server.x509.subject.locality - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.O - target_field: tls.server.x509.subject.organization - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.OU - target_field: tls.server.x509.subject.organizational_unit - ignore_missing: true - - rename: - field: suricata.eve.tls.kv_subject.ST - target_field: tls.server.x509.subject.state_or_province - ignore_missing: true - - set: - field: tls.server.x509.serial_number - value: '{{suricata.eve.tls.serial}}' - ignore_empty_value: true - - gsub: - field: tls.server.x509.serial_number - pattern: ':' - replacement: '' - ignore_missing: true - - date: - field: suricata.eve.tls.notafter - target_field: tls.server.not_after - formats: - - ISO8601 - if: ctx.suricata?.eve?.tls?.notafter != null - - date: - field: suricata.eve.tls.notbefore - target_field: tls.server.not_before - formats: - - ISO8601 - if: ctx.suricata?.eve?.tls?.notbefore != null - - set: - field: tls.server.x509.not_after - value: '{{tls.server.not_after}}' - ignore_empty_value: true - - set: - field: tls.server.x509.not_before - value: '{{tls.server.not_before}}' - ignore_empty_value: true - append: field: related.hosts value: '{{url.domain}}' if: ctx.url?.domain != null && ctx.url?.domain != '' allow_duplicates: false + - append: + if: ctx?.source?.ip != null + field: related.ip + value: '{{source.ip}}' + allow_duplicates: false + - append: + if: ctx?.destination?.ip != null + field: related.ip + value: '{{destination.ip}}' + allow_duplicates: false - remove: field: - suricata.eve.app_proto @@ -345,8 +385,6 @@ processors: - suricata.eve.flow.start - suricata.eve.http.http_method - suricata.eve.http.http_user_agent - - suricata.eve.tls.kv_issuerdn - - suricata.eve.tls.kv_subject ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/suricata/eve/ingest/tls.yml b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml new file mode 100644 index 00000000000..2c84e0c1cb7 --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/tls.yml @@ -0,0 +1,188 @@ +--- +description: Pipeline for Suricata TLS Events + +processors: + - dissect: + field: suricata.eve.tls.version + pattern: '%{tls.version_protocol} %{tls.version}' + ignore_missing: true + - lowercase: + field: tls.version_protocol + ignore_missing: true + - script: + if: ctx?.suricata?.eve?.tls?.sni != null + tag: suricata_trim_tls_sni + lang: painless + source: | + def sni = ctx.suricata.eve.tls.sni; + if (!sni.endsWith(".")) { + return; + } + ctx.suricata.eve.tls.sni = sni.substring(0, sni.length() - 1); + # Subject + - set: + field: tls.server.subject + value: '{{suricata.eve.tls.subject}}' + ignore_empty_value: true + - gsub: + field: suricata.eve.tls.subject + pattern: \\, + replacement: "" + ignore_missing: true + - kv: + field: suricata.eve.tls.subject + field_split: ', ' + value_split: '=' + target_field: suricata.eve.tls.kv_subject + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.C + target_field: tls.server.x509.subject.country + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.CN + target_field: tls.server.x509.subject.common_name + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.L + target_field: tls.server.x509.subject.locality + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.O + target_field: tls.server.x509.subject.organization + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.OU + target_field: tls.server.x509.subject.organizational_unit + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_subject.ST + target_field: tls.server.x509.subject.state_or_province + ignore_missing: true + # Issuer + - set: + field: tls.server.issuer + value: '{{suricata.eve.tls.issuerdn}}' + ignore_empty_value: true + - gsub: + field: suricata.eve.tls.issuerdn + pattern: \\, + replacement: "" + ignore_missing: true + - kv: + field: suricata.eve.tls.issuerdn + field_split: ', ' + value_split: '=' + target_field: suricata.eve.tls.kv_issuerdn + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.C + target_field: tls.server.x509.issuer.country + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.CN + target_field: tls.server.x509.issuer.common_name + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.L + target_field: tls.server.x509.issuer.locality + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.O + target_field: tls.server.x509.issuer.organization + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.OU + target_field: tls.server.x509.issuer.organizational_unit + ignore_missing: true + - rename: + field: suricata.eve.tls.kv_issuerdn.ST + target_field: tls.server.x509.issuer.state_or_province + ignore_missing: true + + - convert: + field: suricata.eve.tls.session_resumed + target_field: tls.resumed + type: boolean + ignore_missing: true + - set: + field: tls.server.hash.sha1 + value: '{{suricata.eve.tls.fingerprint}}' + ignore_empty_value: true + - uppercase: + field: tls.server.hash.sha1 + ignore_missing: true + - split: + field: tls.server.hash.sha1 + separator: ":" + ignore_missing: true + - join: + field: tls.server.hash.sha1 + separator: "" + ignore_failure: true + - append: + field: related.hash + value: "{{tls.server.hash.sha1}}" + if: "ctx?.tls?.server?.hash?.sha1 != null" + - set: + field: tls.client.server_name + value: '{{suricata.eve.tls.sni}}' + ignore_empty_value: true + - set: + field: destination.domain + value: '{{suricata.eve.tls.sni}}' + ignore_empty_value: true + - set: + field: tls.server.ja3s + value: '{{suricata.eve.tls.ja3s.hash}}' + ignore_empty_value: true + - set: + field: tls.client.ja3 + value: '{{suricata.eve.tls.ja3.hash}}' + ignore_empty_value: true + - set: + field: tls.server.certificate + value: '{{suricata.eve.tls.certificate}}' + ignore_empty_value: true + - set: + field: tls.server.certificate_chain + value: '{{suricata.eve.tls.chain}}' + ignore_empty_value: true + - set: + field: tls.server.x509.serial_number + value: '{{suricata.eve.tls.serial}}' + ignore_empty_value: true + - gsub: + field: tls.server.x509.serial_number + pattern: ':' + replacement: '' + ignore_missing: true + - date: + field: suricata.eve.tls.notafter + target_field: tls.server.not_after + formats: + - ISO8601 + if: ctx.suricata?.eve?.tls?.notafter != null + - date: + field: suricata.eve.tls.notbefore + target_field: tls.server.not_before + formats: + - ISO8601 + if: ctx.suricata?.eve?.tls?.notbefore != null + - set: + field: tls.server.x509.not_after + value: '{{tls.server.not_after}}' + ignore_empty_value: true + - set: + field: tls.server.x509.not_before + value: '{{tls.server.not_before}}' + ignore_empty_value: true + - remove: + field: + - suricata.eve.tls.kv_issuerdn + - suricata.eve.tls.kv_subject + ignore_missing: true +on_failure: + - append: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/suricata/eve/manifest.yml b/x-pack/filebeat/module/suricata/eve/manifest.yml index 99efdd65ed4..8d7f87d8373 100644 --- a/x-pack/filebeat/module/suricata/eve/manifest.yml +++ b/x-pack/filebeat/module/suricata/eve/manifest.yml @@ -13,7 +13,12 @@ var: - name: community_id default: true -ingest_pipeline: ingest/pipeline.yml +ingest_pipeline: + - ingest/pipeline.yml + - ingest/dns.yml + - ingest/dns-answer-v1.yml + - ingest/dns-answer-v2.yml + - ingest/tls.yml input: config/eve.yml requires.processors: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json index a36d9d951ad..7201aa1509c 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-dns-4.1.4.log-expected.json @@ -137,6 +137,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2607:f8b0:4006:0805:0000:0000:0000:200e", "10.0.2.3", "10.0.2.15" ], @@ -200,6 +201,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "172.217.11.46", "10.0.2.3", "10.0.2.15" ], @@ -384,6 +386,10 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "151.101.130.217", + "151.101.194.217", + "151.101.2.217", + "151.101.66.217", "10.0.2.3", "10.0.2.15" ], @@ -474,6 +480,10 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2a04:4e42:0600:0000:0000:0000:0000:0729", + "2a04:4e42:0000:0000:0000:0000:0000:0729", + "2a04:4e42:0200:0000:0000:0000:0000:0729", + "2a04:4e42:0400:0000:0000:0000:0000:0729", "10.0.2.3", "10.0.2.15" ], @@ -600,6 +610,10 @@ "type": "CNAME" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "28329", "dns.response_code": "NOERROR", "dns.type": "answer", @@ -654,7 +668,14 @@ "type": "A" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "28329", + "dns.resolved_ip": [ + "98.138.219.232" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -674,6 +695,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "98.138.219.232", "10.0.2.3", "10.0.2.15" ], @@ -708,7 +730,14 @@ "type": "A" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "28329", + "dns.resolved_ip": [ + "98.138.219.231" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -728,6 +757,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "98.138.219.231", "10.0.2.3", "10.0.2.15" ], @@ -762,7 +792,14 @@ "type": "A" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "28329", + "dns.resolved_ip": [ + "72.30.35.10" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -782,6 +819,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "72.30.35.10", "10.0.2.3", "10.0.2.15" ], @@ -816,7 +854,14 @@ "type": "A" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "28329", + "dns.resolved_ip": [ + "72.30.35.9" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -836,6 +881,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "72.30.35.9", "10.0.2.3", "10.0.2.15" ], @@ -870,6 +916,10 @@ "type": "CNAME" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "7050", "dns.response_code": "NOERROR", "dns.type": "answer", @@ -924,7 +974,14 @@ "type": "AAAA" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "7050", + "dns.resolved_ip": [ + "2001:4998:0058:1836:0000:0000:0000:0010" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -944,6 +1001,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2001:4998:0058:1836:0000:0000:0000:0010", "10.0.2.3", "10.0.2.15" ], @@ -978,7 +1036,14 @@ "type": "AAAA" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "7050", + "dns.resolved_ip": [ + "2001:4998:0044:041d:0000:0000:0000:0003" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -998,6 +1063,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2001:4998:0044:041d:0000:0000:0000:0003", "10.0.2.3", "10.0.2.15" ], @@ -1032,7 +1098,14 @@ "type": "AAAA" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "7050", + "dns.resolved_ip": [ + "2001:4998:0058:1836:0000:0000:0000:0011" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -1052,6 +1125,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2001:4998:0058:1836:0000:0000:0000:0011", "10.0.2.3", "10.0.2.15" ], @@ -1086,7 +1160,14 @@ "type": "AAAA" } ], + "dns.header_flags": [ + "RD", + "RA" + ], "dns.id": "7050", + "dns.resolved_ip": [ + "2001:4998:0044:041d:0000:0000:0000:0004" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -1106,6 +1187,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2001:4998:0044:041d:0000:0000:0000:0004", "10.0.2.3", "10.0.2.15" ], @@ -1292,6 +1374,10 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "151.101.194.217", + "151.101.2.217", + "151.101.66.217", + "151.101.130.217", "10.0.2.3", "10.0.2.15" ], @@ -1382,6 +1468,10 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "2a04:4e42:0000:0000:0000:0000:0000:0729", + "2a04:4e42:0200:0000:0000:0000:0000:0729", + "2a04:4e42:0400:0000:0000:0000:0000:0729", + "2a04:4e42:0600:0000:0000:0000:0000:0729", "10.0.2.3", "10.0.2.15" ], diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 2db09a8ee38..e4de4700be8 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -236,6 +236,9 @@ } ], "dns.id": "12308", + "dns.resolved_ip": [ + "172.217.13.110" + ], "dns.response_code": "NOERROR", "dns.type": "answer", "event.category": [ @@ -255,6 +258,7 @@ "network.protocol": "dns", "network.transport": "udp", "related.ip": [ + "172.217.13.110", "192.168.86.1", "192.168.86.85" ], @@ -278,6 +282,9 @@ }, { "@timestamp": "2018-07-05T19:51:23.009Z", + "event.category": [ + "network" + ], "event.dataset": "suricata.eve", "event.kind": "metric", "event.module": "suricata",