From 2d402b80657ab4542e962e0fa47c12875aef1291 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 17:18:54 -0600 Subject: [PATCH 01/40] Adds basic integration test for threat enrichment --- .../tests/create_threat_matching.ts | 54 +++- .../filebeat/threat_intel/data.json | 139 ++++++++++ .../filebeat/threat_intel/mappings.json | 243 +++++++++++++++++ .../filebeat/threat_intel_nested/data.json | 139 ++++++++++ .../threat_intel_nested/mappings.json | 244 ++++++++++++++++++ 5 files changed, 818 insertions(+), 1 deletion(-) create mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel/data.json create mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json create mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json create mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 9b29548cbe19ef..ee8a02ba95c419 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -90,7 +90,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe('tests with auditbeat data', () => { + describe.only('tests with auditbeat data', () => { beforeEach(async () => { await deleteAllAlerts(supertest); await createSignalsIndex(supertest); @@ -251,6 +251,58 @@ export default ({ getService }: FtrProviderContext) => { const signalsOpen = await getSignalsByIds(supertest, [ruleResponse.id]); expect(signalsOpen.hits.hits.length).equal(0); }); + + describe.only('indicator enrichment', () => { + beforeEach(async () => { + await esArchiver.load('filebeat/threat_intel'); + }); + + afterEach(async () => { + await esArchiver.unload('filebeat/threat_intel'); + }); + + it('enriches signals with the single indicator that matched', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'threat.indicator.domain: *', // narrow things down to indicators with a domain + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(2); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([{}, {}]); + }); + + it('enriches signals with multiple indicators if several matched'); + it('enriches signals with all fields that the indicator matched'); + }); }); }); }; diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json new file mode 100644 index 00000000000000..127d4bb750e771 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json @@ -0,0 +1,139 @@ +{ + "type": "doc", + "value": { + "id": "978783", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.595350Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978783/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "domain": "159.89.119.67", + "first_seen": "2021-01-26T11:09:04.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://159.89.119.67:59600/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": false, + "tags": null, + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978782", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "domain": "125.46.136.106", + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://125.46.136.106:52014/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json new file mode 100644 index 00000000000000..26d8e29eaecf77 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/mappings.json @@ -0,0 +1,243 @@ +{ + "type": "index", + "value": { + "aliases": {}, + "index": "filebeat-8.0.0-2021.01.26-000001", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.0.0" + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "indicator": { + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "wildcard" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "first_seen": { + "type": "date" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "last_seen": { + "type": "date" + }, + "marking": { + "properties": { + "tlp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "matched": { + "properties": { + "atomic": { + "ignore_above": 1024, + "type": "keyword" + }, + "field": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner_stats": { + "type": "long" + }, + "sightings": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": "filebeat-8.0.0", + "rollover_alias": "filebeat-filebeat-8.0.0" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json new file mode 100644 index 00000000000000..127d4bb750e771 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json @@ -0,0 +1,139 @@ +{ + "type": "doc", + "value": { + "id": "978783", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.595350Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978783/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "domain": "159.89.119.67", + "first_seen": "2021-01-26T11:09:04.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://159.89.119.67:59600/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": false, + "tags": null, + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978782", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "domain": "125.46.136.106", + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://125.46.136.106:52014/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json new file mode 100644 index 00000000000000..73f9b5ba1934f7 --- /dev/null +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json @@ -0,0 +1,244 @@ +{ + "type": "index", + "value": { + "aliases": {}, + "index": "filebeat-8.0.0-2021.01.26-000001", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.0.0" + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "@version": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "indicator": { + "type": "nested", + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "wildcard" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "first_seen": { + "type": "date" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "last_seen": { + "type": "date" + }, + "marking": { + "properties": { + "tlp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "matched": { + "properties": { + "atomic": { + "ignore_above": 1024, + "type": "keyword" + }, + "field": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner_stats": { + "type": "long" + }, + "sightings": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "settings": { + "index": { + "lifecycle": { + "name": "filebeat-8.0.0", + "rollover_alias": "filebeat-filebeat-8.0.0" + }, + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "5s" + } + } + } +} From 876b49d602d7fca464ee653e6eebc98e5e0904fe Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 17:24:10 -0600 Subject: [PATCH 02/40] Update signals mappings with indicator fields --- .../routes/index/ecs_mapping.json | 160 ++++++++++++++++++ .../routes/index/get_signals_template.ts | 2 +- 2 files changed, 161 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json index 6126ee462ec202..b4e6e4d1400c50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -2457,6 +2457,144 @@ "ignore_above": 1024, "type": "keyword" }, + "indicator": { + "type": "nested", + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "type": "wildcard" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "first_seen": { + "type": "date" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "last_seen": { + "type": "date" + }, + "marking": { + "properties": { + "tlp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "matched": { + "properties": { + "atomic": { + "ignore_above": 1024, + "type": "keyword" + }, + "field": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner_stats": { + "type": "long" + }, + "sightings": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "tactic": { "properties": { "id": { @@ -2492,6 +2630,28 @@ "reference": { "ignore_above": 1024, "type": "keyword" + }, + "subtechnique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } } } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index c9a4f168224d43..83992f2b3f1c3f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -8,7 +8,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 14; +export const SIGNALS_TEMPLATE_VERSION = 15; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { From 9f9922d5f9bc552208c1ee3c21025d4fe2861119 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 21 Jan 2021 17:28:55 -0600 Subject: [PATCH 03/40] Simplify some ternaries with Math.min --- .../lib/detection_engine/signals/search_after_bulk_create.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index b821909ca907c7..9907d9f46186d6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -106,7 +106,7 @@ export const searchAfterAndBulkCreate = async ({ services, logger, filter, - pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, excludeDocsWithTimestampOverride: true, }); @@ -149,7 +149,7 @@ export const searchAfterAndBulkCreate = async ({ services, logger, filter, - pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result. + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), timestampOverride: ruleParams.timestampOverride, excludeDocsWithTimestampOverride: false, }); From 9620fa8fee32e63063644f37339df306b7ef3009 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 21 Jan 2021 17:29:26 -0600 Subject: [PATCH 04/40] Remove outdated comments --- .../detection_engine/signals/search_after_bulk_create.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 9907d9f46186d6..eb2a6fa3d08035 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -139,7 +139,6 @@ export const searchAfterAndBulkCreate = async ({ } if (hasSortId) { - // only execute search if we have something to sort on or if it is the first search const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ buildRuleMessage, searchAfterSortId: sortId, @@ -186,14 +185,6 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) ); - // search results yielded zero hits so exit - // with search_after, these two values can be different when - // searching with the last sortId of a consecutive search_after - // yields zero hits, but there were hits using the previous - // sortIds. - // e.g. totalHits was 156, index 50 of 100 results, do another search-after - // this time with a new sortId, index 22 of the remaining 56, get another sortId - // search with that sortId, total is still 156 but the hits.hits array is empty. if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { logger.debug( buildRuleMessage( From 221111054c2ded1de4a3318ed4e6282f2234b6bc Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 21 Jan 2021 17:30:12 -0600 Subject: [PATCH 05/40] Add notes from walkthrough with devin --- .../lib/detection_engine/signals/search_after_bulk_create.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index eb2a6fa3d08035..09d3db05b3a8d1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -117,6 +117,8 @@ export const searchAfterAndBulkCreate = async ({ backupSortId = lastSortId[0]; hasBackupSortId = true; } else { + // TODO: This comment does not seem to match the code; this is the first loop and the first search, so how could the initial search result be known? + // answer: these were previously not blocking searches, and so the "second" search actually executed first TLDR outdated comment // if no sort id on backup search and the initial search result was also empty logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB')); hasBackupSortId = false; @@ -124,6 +126,7 @@ export const searchAfterAndBulkCreate = async ({ mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]); + // TODO again, on the first loop this is the FIRST search and toReturn is default values // merge the search result from the secondary search with the first toReturn = mergeReturns([ toReturn, @@ -165,6 +168,7 @@ export const searchAfterAndBulkCreate = async ({ }), ]); + // TODO it's unclear why hits are guaranteed here; the type appears to be identical to the previous result which has more guards // we are guaranteed to have searchResult hits at this point // because we check before if the totalHits or // searchResult.hits.hits.length is 0 From e12c529afd5ea3543be45ba814b6bad141a62c70 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 18:27:39 -0600 Subject: [PATCH 06/40] Add an enrichment hook to the current signal creation pipeline When this moves to individual rule-specific data transformations this will be a little more explicit/configurable; for now to keep changes minimal, we're using dependency injection to pass a function, which will default to the identity function (e.g. a no-op). --- .../detection_engine/signals/search_after_bulk_create.ts | 6 +++++- .../server/lib/detection_engine/signals/types.ts | 3 +++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 09d3db05b3a8d1..f78c9e20251568 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -7,6 +7,7 @@ /* eslint-disable complexity */ +import { identity } from 'lodash'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filters/filter_events_against_list'; @@ -49,6 +50,7 @@ export const searchAfterAndBulkCreate = async ({ tags, throttle, buildRuleMessage, + enrichment = identity, }: SearchAfterAndBulkCreateParams): Promise => { let toReturn = createSearchAfterReturnType(); @@ -223,6 +225,8 @@ export const searchAfterAndBulkCreate = async ({ tuple.maxSignals - signalsCreatedCount ); } + const enrichedEvents = await enrichment(filteredEvents); + const { bulkCreateDuration: bulkDuration, createdItemsCount: createdCount, @@ -231,7 +235,7 @@ export const searchAfterAndBulkCreate = async ({ errors: bulkErrors, } = await singleBulkCreate({ buildRuleMessage, - filteredEvents, + filteredEvents: enrichedEvents, ruleParams, services, logger, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 8031b81f70eb05..f7ac0425b2f2e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -228,6 +228,8 @@ export interface QueryFilter { }; } +export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise; + export interface SearchAfterAndBulkCreateParams { gap: moment.Duration | null; previousStartedAt: Date | null | undefined; @@ -254,6 +256,7 @@ export interface SearchAfterAndBulkCreateParams { tags: string[]; throttle: string; buildRuleMessage: BuildRuleMessage; + enrichment?: SignalsEnrichment; } export interface SearchAfterAndBulkCreateReturnType { From 0b1dd8b7d5fffe6f6291d42893e8bc583cd5c6e9 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 19:01:29 -0600 Subject: [PATCH 07/40] Add utility functions for encoding/decoding our threat query This is what allows us to enrich the threat match signals using only the signal search response. --- .../signals/threat_mapping/utils.test.ts | 55 +++++++++++++++++++ .../signals/threat_mapping/utils.ts | 28 +++++++++- 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index a738c8a864a1c6..897143f9ae5744 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -7,6 +7,7 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; import { sampleSignalHit } from '../__mocks__/es_results'; +import { ThreatMatchNamedQuery } from './types'; import { calculateAdditiveMax, @@ -14,6 +15,8 @@ import { calculateMaxLookBack, combineConcurrentResults, combineResults, + decodeThreatMatchNamedQuery, + encodeThreatMatchNamedQuery, } from './utils'; describe('utils', () => { @@ -580,4 +583,56 @@ describe('utils', () => { ); }); }); + + describe('threat match queries', () => { + describe('encodeThreatMatchNamedQuery()', () => { + it('generates a string that can be later decoded', () => { + const encoded = encodeThreatMatchNamedQuery({ + id: 'id', + field: 'field', + value: 'value', + }); + + expect(typeof encoded).toEqual('string'); + }); + }); + + describe('decodeThreatMatchNamedQuery()', () => { + it('can decode an encoded query', () => { + const query: ThreatMatchNamedQuery = { + id: 'my_id', + field: 'threat.indicator.domain', + value: 'host.name', + }; + + const encoded = encodeThreatMatchNamedQuery(query); + const decoded = decodeThreatMatchNamedQuery(encoded); + + expect(decoded).not.toBe(query); + expect(decoded).toEqual(query); + }); + + it('raises an error if the input is invalid', () => { + const badInput = 'nope'; + + expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( + 'Decoded query is invalid. Decoded value: {"id":"nope"}' + ); + }); + + it('raises an error if the query is missing a value', () => { + const badQuery: ThreatMatchNamedQuery = { + id: 'my_id', + // @ts-expect-error field intentionally undefined + field: undefined, + value: 'host.name', + }; + const badInput = encodeThreatMatchNamedQuery(badQuery); + + expect(() => decodeThreatMatchNamedQuery(badInput)).toThrowError( + 'Decoded query is invalid. Decoded value: {"id":"my_id","field":"","value":"host.name"}' + ); + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 87bcb657a53a52..72d9257798e1c9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { SearchAfterAndBulkCreateReturnType, SignalSourceHit } from '../types'; +import { ThreatMatchNamedQuery } from './types'; /** * Given two timers this will take the max of each and add them to each other and return that addition. @@ -113,3 +114,28 @@ export const combineConcurrentResults = ( return combineResults(currentResult, maxedNewResult); }; + +const separator = '___SEPARATOR___'; +export const encodeThreatMatchNamedQuery = ({ + id, + field, + value, +}: ThreatMatchNamedQuery): string => { + return [id, field, value].join(separator); +}; + +export const decodeThreatMatchNamedQuery = (encoded: string): ThreatMatchNamedQuery => { + const queryValues = encoded.split(separator); + const [id, field, value] = queryValues; + const query = { id, field, value }; + + if (queryValues.length !== 3 || !queryValues.every(Boolean)) { + const queryString = JSON.stringify(query); + throw new Error(`Decoded query is invalid. Decoded value: ${queryString}`); + } + + return query; +}; + +export const extractNamedQueries = (hit: SignalSourceHit): ThreatMatchNamedQuery[] => + hit.matched_queries?.map((match) => decodeThreatMatchNamedQuery(match)) ?? []; From 732a525b99f0372d0f6170aba4669e1d505d0ea8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 23:14:14 -0600 Subject: [PATCH 08/40] Add a name to each threat match filter clause This gives us the information we need to enrich our signals after they've been queried without having to perform a complicated reverse query. --- .../threat_mapping/build_threat_mapping_filter.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts index 180895877bdd22..885ee4a44a0241 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.ts @@ -17,6 +17,7 @@ import { FilterThreatMappingOptions, SplitShouldClausesOptions, } from './types'; +import { encodeThreatMatchNamedQuery } from './utils'; export const MAX_CHUNK_SIZE = 1024; @@ -78,7 +79,14 @@ export const createInnerAndClauses = ({ should: [ { match: { - [threatMappingEntry.field]: value, + [threatMappingEntry.field]: { + query: value, + _name: encodeThreatMatchNamedQuery({ + id: threatListItem._id, + field: threatMappingEntry.field, + value: threatMappingEntry.value, + }), + }, }, }, ], From 8fdf571399c281bb0bb28f021d64d745883f90e0 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 23:21:46 -0600 Subject: [PATCH 09/40] Adds functions for signal enrichment of threat indicators --- .../build_threat_mapping_filter.mock.ts | 3 +- .../enrich_signal_threat_matches.mock.ts | 35 +++ .../enrich_signal_threat_matches.test.ts | 272 ++++++++++++++++++ .../enrich_signal_threat_matches.ts | 78 +++++ .../signals/threat_mapping/types.ts | 12 + 5 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index a88d9061f7a1ff..87a5f8143dfe8e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -86,7 +86,7 @@ export const getThreatListSearchResponseMock = (): SearchResponse ({ +export const getThreatListItemMock = (overrides: Partial = {}): ThreatListItem => ({ '@timestamp': '2020-09-09T21:59:13Z', host: { name: 'host-1', @@ -100,6 +100,7 @@ export const getThreatListItemMock = (): ThreatListItem => ({ ip: '127.0.0.1', port: 1, }, + ...overrides, }); export const getFilterThreatMapping = (): ThreatMapping => [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts new file mode 100644 index 00000000000000..d694ba3e532b0c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SignalSearchResponse, SignalSourceHit } from '../types'; +import { ThreatMatchNamedQuery } from './types'; + +export const getNamedQueryMock = ( + overrides: Partial = {} +): ThreatMatchNamedQuery => ({ + id: 'id', + field: 'field', + value: 'value', + ...overrides, +}); + +export const getSignalHitMock = (overrides: Partial = {}): SignalSourceHit => ({ + _id: '_id', + _index: '_index', + _source: { + '@timestamp': '2020-11-20T15:35:28.373Z', + }, + _type: '_type', + _score: 0, + ...overrides, +}); + +export const getSignalsResponseMock = (signals: SignalSourceHit[] = []): SignalSearchResponse => ({ + took: 1, + timed_out: false, + _shards: { total: 1, successful: 1, skipped: 0, failed: 0 }, + hits: { total: { value: signals.length, relation: 'eq' }, max_score: 0, hits: signals }, +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts new file mode 100644 index 00000000000000..0448fb93cfc0cb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -0,0 +1,272 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; + +import { getThreatListItemMock } from './build_threat_mapping_filter.mock'; +import { buildMatchedIndicator, enrichSignalThreatMatches } from './enrich_signal_threat_matches'; +import { + getNamedQueryMock, + getSignalHitMock, + getSignalsResponseMock, +} from './enrich_signal_threat_matches.mock'; +import { GetMatchedThreats, ThreatListItem, ThreatMatchNamedQuery } from './types'; +import { encodeThreatMatchNamedQuery } from './utils'; + +describe('buildMatchedIndicator', () => { + let threats: ThreatListItem[]; + let queries: ThreatMatchNamedQuery[]; + + beforeEach(() => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + queries = [getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' })]; + }); + + it('returns an empty list if queries is empty', () => { + const indicators = buildMatchedIndicator({ + queries: [], + threats, + }); + + expect(indicators).toEqual([]); + }); + + it('returns the value of the matched indicator as matched.atomic', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.atomic')).toEqual('domain_1'); + }); + + it('returns the field of the matched indicator as matched.field', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.field')).toEqual('threat.indicator.domain'); + }); + + it('returns the type of the matched indicator as matched.type', () => { + const [indicator] = buildMatchedIndicator({ + queries, + threats, + }); + + expect(get(indicator, 'matched.type')).toEqual('type_1'); + }); + + it('returns indicators for each provided query', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + getThreatListItemMock({ + _id: '456', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + queries = [ + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }), + getNamedQueryMock({ id: '456', value: 'threat.indicator.other' }), + getNamedQueryMock({ id: '456', value: 'threat.indicator.domain' }), + ]; + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toHaveLength(queries.length); + }); + + it('returns the indicator data the specified at threat.indicator by default', () => { + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + domain: 'domain_1', + matched: { + atomic: 'domain_1', + field: 'threat.indicator.domain', + type: 'type_1', + }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it('returns the indicator data the specified at the custom path', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + 'threat.indicator.domain': 'domain_1', + custom: { + indicator: { + path: { + indicator_field: 'indicator_field_1', + type: 'indicator_type', + }, + }, + }, + }, + }), + ]; + + const indicators = buildMatchedIndicator({ + indicatorPath: 'custom.indicator.path', + queries, + threats, + }); + + expect(indicators).toEqual([ + { + indicator_field: 'indicator_field_1', + matched: { + atomic: 'domain_1', + field: 'threat.indicator.domain', + type: 'indicator_type', + }, + type: 'indicator_type', + }, + ]); + }); +}); + +describe('enrichSignalThreatMatches', () => { + let getMatchedThreats: GetMatchedThreats; + + beforeEach(() => { + getMatchedThreats = async () => [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + ]; + }); + + it('performs no enrichment if there are no signals', async () => { + const signals = getSignalsResponseMock([]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + + expect(enrichedSignals.hits.hits).toEqual([]); + }); + + it('preserves existing threat.indicator objects on signals', async () => { + const query = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) + ); + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: { indicator: [{ existing: 'indicator' }] } }, + matched_queries: [query], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { existing: 'indicator' }, + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it.skip('preserves an existing threat.indicator object on signals', async () => { + const query = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) + ); + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: { indicator: { existing: 'indicator' } } }, + matched_queries: [query], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { existing: 'indicator' }, + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); + + it('merges duplicate matched signals into a single signal with multiple indicators', async () => { + getMatchedThreats = async () => [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { indicator: { domain: 'domain_1', other: 'other_1', type: 'type_1' } }, + }, + }), + getThreatListItemMock({ + _id: '456', + _source: { + threat: { indicator: { domain: 'domain_2', other: 'other_2', type: 'type_2' } }, + }, + }), + ]; + const query = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) + ); + const otherQuery = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '456', value: 'threat.indicator.domain' }) + ); + const signalHit = getSignalHitMock({ + _id: 'signal123', + matched_queries: [query], + }); + const otherSignalHit = getSignalHitMock({ + _id: 'signal123', + matched_queries: [otherQuery], + }); + const signals = getSignalsResponseMock([signalHit, otherSignalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + expect(enrichedSignals.hits.hits).toHaveLength(1); + + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { existing: 'indicator' }, + { + domain: 'domain_1', + matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + other: 'other_1', + type: 'type_1', + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts new file mode 100644 index 00000000000000..8e5afb330b12c3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get } from 'lodash'; + +import type { SignalSearchResponse } from '../types'; +import type { + GetMatchedThreats, + ThreatIndicator, + ThreatListItem, + ThreatMatchNamedQuery, +} from './types'; +import { extractNamedQueries } from './utils'; + +export const buildMatchedIndicator = ({ + queries, + threats, + indicatorPath = 'threat.indicator', +}: { + queries: ThreatMatchNamedQuery[]; + threats: ThreatListItem[]; + indicatorPath?: string; +}): ThreatIndicator[] => + queries.map((query) => { + const matchedThreat = threats.find((threat) => threat._id === query.id); + // TODO what if this is an array? Grab the first? + const indicator = get(matchedThreat?._source, indicatorPath); + const atomic = get(matchedThreat?._source, query.value); + const type = get(indicator, 'type'); + // console.log('matchedThreat', JSON.stringify(matchedThreat, null, 2)); + // console.log('indicator', JSON.stringify(indicator, null, 2)); + + return { + ...indicator, + matched: { atomic, field: query.value, type }, + }; + }); + +export const enrichSignalThreatMatches = async ( + signals: SignalSearchResponse, + getMatchedThreats: GetMatchedThreats +): Promise => { + const signalHits = signals.hits.hits; + if (signalHits.length === 0) { + return signals; + } + // TODO: merge duplicate signals + + const matches = signalHits.map((signalHit) => extractNamedQueries(signalHit)); + const matchedThreatIds = [...new Set(matches.flat().map(({ id }) => id))]; + const matchedThreats = await getMatchedThreats(matchedThreatIds); + const matchedIndicators = matches.map((queries) => + buildMatchedIndicator({ queries, threats: matchedThreats }) + ); + + const enrichedSignals = signalHits.map((signalHit, i) => { + const threat = get(signalHit._source, 'threat') ?? {}; + const existingIndicators = get(signalHit._source, 'threat.indicator') ?? []; + + return { + ...signalHit, + _source: { + ...signalHit._source, + threat: { + ...threat, + indicator: [...existingIndicators, ...matchedIndicators[i]], + }, + }, + }; + }); + // eslint-disable-next-line require-atomic-updates + signals.hits.hits = enrichedSignals; + + return signals; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 26e42b795be3ea..671d948a38d422 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -185,6 +185,18 @@ export interface ThreatListItem { [key: string]: unknown; } +export interface ThreatIndicator { + [key: string]: unknown; +} + export interface SortWithTieBreaker { [key: string]: string; } + +export interface ThreatMatchNamedQuery { + id: string; + field: string; + value: string; +} + +export type GetMatchedThreats = (ids: string[]) => Promise; From 97122501946dffbd2fcc95764710482a46f8354f Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Sun, 31 Jan 2021 23:24:28 -0600 Subject: [PATCH 10/40] Wire up threat enrichment to threat match rules --- .../threat_mapping/create_threat_signal.ts | 3 ++ .../threat_mapping/create_threat_signals.ts | 39 ++++++++++++++++++- .../signals/threat_mapping/types.ts | 3 +- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index a076ab46aae2a6..ba428bc0771250 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -14,6 +14,7 @@ import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ threatMapping, + threatEnrichment, query, inputIndex, type, @@ -77,6 +78,7 @@ export const createThreatSignal = async ({ `${threatFilter.query.bool.should.length} indicator items are being checked for existence of matches` ) ); + const result = await searchAfterAndBulkCreate({ gap, previousStartedAt, @@ -103,6 +105,7 @@ export const createThreatSignal = async ({ tags, throttle, buildRuleMessage, + enrichment: threatEnrichment, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 1e486e58aa0730..9b29efbfd6b877 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -8,10 +8,11 @@ import chunk from 'lodash/fp/chunk'; import { getThreatList, getThreatListCount } from './get_threat_list'; -import { CreateThreatSignalsOptions } from './types'; +import { CreateThreatSignalsOptions, GetMatchedThreats } from './types'; import { createThreatSignal } from './create_threat_signal'; -import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { SearchAfterAndBulkCreateReturnType, SignalSearchResponse } from '../types'; import { combineConcurrentResults } from './utils'; +import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; export const createThreatSignals = async ({ threatMapping, @@ -90,12 +91,46 @@ export const createThreatSignals = async ({ perPage, }); + const getMatchedThreats: GetMatchedThreats = async (ids) => { + // TODO should _id be a configurable field? + const matchedThreatsFilter = { + query: { + bool: { + filter: { + ids: { values: ids }, + }, + }, + }, + }; + const threatResponse = await getThreatList({ + callCluster: services.callCluster, + exceptionItems: exceptionItems ?? [], + threatFilters: [...(threatFilters ?? []), matchedThreatsFilter], + query: threatQuery, + language: threatLanguage, + index: threatIndex, + listClient, + searchAfter: undefined, + sortField: undefined, + sortOrder: undefined, + logger, + buildRuleMessage, + perPage: undefined, + }); + + return threatResponse.hits.hits ?? []; + }; + + const threatEnrichment = (signals: SignalSearchResponse): Promise => + enrichSignalThreatMatches(signals, getMatchedThreats); + while (threatList.hits.hits.length !== 0) { const chunks = chunk(itemsPerSearch, threatList.hits.hits); logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ + threatEnrichment, threatMapping, query, inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 671d948a38d422..44cf18ab75d704 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -31,7 +31,7 @@ import { ILegacyScopedClusterClient, Logger } from '../../../../../../../../src/ import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; -import { SearchAfterAndBulkCreateReturnType } from '../types'; +import { SearchAfterAndBulkCreateReturnType, SignalsEnrichment } from '../types'; export type SortOrderOrUndefined = 'asc' | 'desc' | undefined; @@ -76,6 +76,7 @@ export interface CreateThreatSignalsOptions { export interface CreateThreatSignalOptions { threatMapping: ThreatMapping; + threatEnrichment: SignalsEnrichment; query: string; inputIndex: string[]; type: Type; From 7936cb08b4818e773dfb733d2aa1c0d9482f370b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 1 Feb 2021 00:24:33 -0600 Subject: [PATCH 11/40] Fleshes out threat match integration tests Adds assertions to the existing test, and fleshes out another test for a multi-match signal. --- .../tests/create_threat_matching.ts | 106 ++++++++++++- .../filebeat/threat_intel/data.json | 142 +++++++++++++++++- 2 files changed, 244 insertions(+), 4 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index ee8a02ba95c419..2e796ad325a23c 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -297,11 +297,111 @@ export default ({ getService }: FtrProviderContext) => { const { hits } = signalsOpen.hits; const threats = hits.map((hit) => hit._source.threat); - expect(threats).to.eql([{}, {}]); + expect(threats).to.eql([ + { + indicator: [ + { + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'threat.indicator.domain', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + { + indicator: [ + { + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'threat.indicator.domain', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + ]); }); - it('enriches signals with multiple indicators if several matched'); - it('enriches signals with all fields that the indicator matched'); + it('enriches signals with multiple indicators if several matched', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + threat_query: 'threat.indicator: *', // narrow things down to indicators with a domain + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + // { + // value: 'threat.indicator.ip', + // field: 'source.ip', + // type: 'mapping', + // }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(1); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + first_seen: '2021-01-26T11:06:03.000Z', + matched: { + atomic: 57324, + field: 'threat.indicator.port', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://193.70.81.238:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + ]); + }); }); }); }); diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json index 127d4bb750e771..a6f38d65cf8717 100644 --- a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json @@ -69,7 +69,7 @@ { "type": "doc", "value": { - "id": "978782", + "id": "978784", "index": "filebeat-8.0.0-2021.01.26-000001", "source": { "@timestamp": "2021-01-26T11:09:05.529Z", @@ -137,3 +137,143 @@ } } } + +{ + "type": "doc", + "value": { + "id": "978785", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "port": 57324, + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "geenensp", + "type": "url", + "url": { + "full": "http://193.70.81.238:59600/bin.sh", + "scheme": "http" + } + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} + +{ + "type": "doc", + "value": { + "id": "978787", + "index": "filebeat-8.0.0-2021.01.26-000001", + "source": { + "@timestamp": "2021-01-26T11:09:05.529Z", + "agent": { + "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", + "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", + "name": "MacBook-Pro-de-Gloria.local", + "type": "filebeat", + "version": "8.0.0" + }, + "ecs": { + "version": "1.6.0" + }, + "event": { + "category": "threat", + "created": "2021-01-26T11:09:05.529Z", + "dataset": "threatintel.abuseurl", + "ingested": "2021-01-26T11:09:06.616763Z", + "kind": "enrichment", + "module": "threatintel", + "reference": "https://urlhaus.abuse.ch/url/978782/", + "type": "indicator" + }, + "fileset": { + "name": "abuseurl" + }, + "input": { + "type": "httpjson" + }, + "service": { + "type": "threatintel" + }, + "tags": [ + "threatintel-abuseurls", + "forwarded" + ], + "threat": { + "indicator": { + "ip": "45.115.45.3", + "first_seen": "2021-01-26T11:06:03.000Z", + "provider": "other_provider", + "type": "ip" + } + }, + "threatintel": { + "abuseurl": { + "blacklists": { + "spamhaus_dbl": "not listed", + "surbl": "not listed" + }, + "larted": true, + "tags": [ + "32-bit", + "elf", + "mips" + ], + "threat": "malware_download", + "url_status": "online" + } + } + } + } +} From 276f1c8fc97218419acd3fd9508f14b72cf2ed3b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 1 Feb 2021 11:31:12 -0600 Subject: [PATCH 12/40] Add more test cases to indicator match integration tests * single indicator matching multiple events * multiple indicators matching a single event * multiple indicators, multiple events * placeholder for deduplication logic This also adds some descriptions to our threat intel documents, to give a little context around how they're meant to function within the tests, particularly as relates to the auditbeat/hosts data on which it is meant to function. --- .../tests/create_threat_matching.ts | 242 +++++++++++++++++- .../filebeat/threat_intel/data.json | 19 +- 2 files changed, 240 insertions(+), 21 deletions(-) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 2e796ad325a23c..9b5e5b588ea281 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -90,7 +90,7 @@ export default ({ getService }: FtrProviderContext) => { }); }); - describe.only('tests with auditbeat data', () => { + describe('tests with auditbeat data', () => { beforeEach(async () => { await deleteAllAlerts(supertest); await createSignalsIndex(supertest); @@ -252,7 +252,7 @@ export default ({ getService }: FtrProviderContext) => { expect(signalsOpen.hits.hits.length).equal(0); }); - describe.only('indicator enrichment', () => { + describe('indicator enrichment', () => { beforeEach(async () => { await esArchiver.load('filebeat/threat_intel'); }); @@ -301,6 +301,7 @@ export default ({ getService }: FtrProviderContext) => { { indicator: [ { + description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', first_seen: '2021-01-26T11:09:04.000Z', matched: { @@ -320,6 +321,7 @@ export default ({ getService }: FtrProviderContext) => { { indicator: [ { + description: "domain should match the auditbeat hosts' data's source.ip", domain: '159.89.119.67', first_seen: '2021-01-26T11:09:04.000Z', matched: { @@ -350,8 +352,77 @@ export default ({ getService }: FtrProviderContext) => { language: 'kuery', rule_id: 'rule-1', from: '1900-01-01T00:00:00.000Z', - query: '*:*', - threat_query: 'threat.indicator: *', // narrow things down to indicators with a domain + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_query: 'threat.indicator.ip: *', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(1); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'threat.indicator.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'threat.indicator.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, + ], + }, + ]); + }); + + it('adds a single indicator that matched multiple fields', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: 'source.port: 57324', // narrow our query to a single record that matches two indicators + threat_query: 'threat.indicator.ip: *', threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module threat_mapping: [ { @@ -361,11 +432,15 @@ export default ({ getService }: FtrProviderContext) => { field: 'source.port', type: 'mapping', }, - // { - // value: 'threat.indicator.ip', - // field: 'source.ip', - // type: 'mapping', - // }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, ], }, ], @@ -380,11 +455,41 @@ export default ({ getService }: FtrProviderContext) => { const { hits } = signalsOpen.hits; const threats = hits.map((hit) => hit._source.threat); + // TODO how should a signal be enriched if a single indicator matches + // on multiple fields? As evidenced below, we currently duplicate the + // indicator with a different matched object. expect(threats).to.eql([ { indicator: [ { + description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'threat.indicator.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'threat.indicator.ip', + type: 'ip', + }, + provider: 'other_provider', + type: 'ip', + }, + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', matched: { atomic: 57324, field: 'threat.indicator.port', @@ -393,15 +498,132 @@ export default ({ getService }: FtrProviderContext) => { port: 57324, provider: 'geenensp', type: 'url', + }, + ], + }, + ]); + }); + + it('generates multiple signals with multiple matches', async () => { + const rule: CreateRulesSchema = { + description: 'Detecting root and admin users', + name: 'Query with a rule id', + severity: 'high', + index: ['auditbeat-*'], + type: 'threat_match', + risk_score: 55, + language: 'kuery', + rule_id: 'rule-1', + from: '1900-01-01T00:00:00.000Z', + query: '*:*', // narrow our query to a single record that matches two indicators + threat_query: '', + threat_index: ['filebeat-*'], // Mimics indicators from the filebeat MISP module + threat_mapping: [ + { + entries: [ + { + value: 'threat.indicator.port', + field: 'source.port', + type: 'mapping', + }, + { + value: 'threat.indicator.ip', + field: 'source.ip', + type: 'mapping', + }, + ], + }, + { + entries: [ + { + value: 'threat.indicator.domain', + field: 'destination.ip', + type: 'mapping', + }, + ], + }, + ], + threat_filters: [], + }; + + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 2, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + expect(signalsOpen.hits.hits.length).equal(2); + + const { hits } = signalsOpen.hits; + const threats = hits.map((hit) => hit._source.threat); + expect(threats).to.eql([ + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'threat.indicator.domain', + type: 'url', + }, + provider: 'geenensp', + type: 'url', + url: { + full: 'http://159.89.119.67:59600/bin.sh', + scheme: 'http', + }, + }, + ], + }, + { + indicator: [ + { + description: "domain should match the auditbeat hosts' data's source.ip", + domain: '159.89.119.67', + first_seen: '2021-01-26T11:09:04.000Z', + matched: { + atomic: '159.89.119.67', + field: 'threat.indicator.domain', + type: 'url', + }, + provider: 'geenensp', + type: 'url', url: { - full: 'http://193.70.81.238:59600/bin.sh', + full: 'http://159.89.119.67:59600/bin.sh', scheme: 'http', }, }, + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: '45.115.45.3', + field: 'threat.indicator.ip', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, + { + description: 'this should match auditbeat/hosts on both port and ip', + first_seen: '2021-01-26T11:06:03.000Z', + ip: '45.115.45.3', + matched: { + atomic: 57324, + field: 'threat.indicator.port', + type: 'url', + }, + port: 57324, + provider: 'geenensp', + type: 'url', + }, ], }, ]); }); + it('deduplicates a signal if it is found in multiple separate query loops'); }); }); }); diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json index a6f38d65cf8717..0cbc7f37bd519a 100644 --- a/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json +++ b/x-pack/test/functional/es_archives/filebeat/threat_intel/data.json @@ -40,6 +40,7 @@ ], "threat": { "indicator": { + "description": "domain should match the auditbeat hosts' data's source.ip", "domain": "159.89.119.67", "first_seen": "2021-01-26T11:09:04.000Z", "provider": "geenensp", @@ -108,14 +109,11 @@ ], "threat": { "indicator": { - "domain": "125.46.136.106", + "description": "this should not match the auditbeat hosts data", + "ip": "125.46.136.106", "first_seen": "2021-01-26T11:06:03.000Z", "provider": "geenensp", - "type": "url", - "url": { - "full": "http://125.46.136.106:52014/bin.sh", - "scheme": "http" - } + "type": "ip" } }, "threatintel": { @@ -180,14 +178,12 @@ ], "threat": { "indicator": { + "description": "this should match auditbeat/hosts on both port and ip", + "ip": "45.115.45.3", "port": 57324, "first_seen": "2021-01-26T11:06:03.000Z", "provider": "geenensp", - "type": "url", - "url": { - "full": "http://193.70.81.238:59600/bin.sh", - "scheme": "http" - } + "type": "url" } }, "threatintel": { @@ -252,6 +248,7 @@ ], "threat": { "indicator": { + "description": "this should match auditbeat/hosts on ip", "ip": "45.115.45.3", "first_seen": "2021-01-26T11:06:03.000Z", "provider": "other_provider", From bb0584c05bacae3973b88a1a8fa36b5bb8a7db7b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 21:17:18 -0600 Subject: [PATCH 13/40] Implement signal deduplification This handles the situation where the indicator match search has returned the same signal multiple times due to the source event matching different indicators in different query batches. In this case, we want to generate a single signal with all matched indicators. --- .../enrich_signal_threat_matches.test.ts | 69 ++++++++++++++++++- .../enrich_signal_threat_matches.ts | 34 +++++++-- 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 0448fb93cfc0cb..b8404a062d4cc5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -7,7 +7,11 @@ import { get } from 'lodash'; import { getThreatListItemMock } from './build_threat_mapping_filter.mock'; -import { buildMatchedIndicator, enrichSignalThreatMatches } from './enrich_signal_threat_matches'; +import { + buildMatchedIndicator, + enrichSignalThreatMatches, + groupAndMergeSignalMatches, +} from './enrich_signal_threat_matches'; import { getNamedQueryMock, getSignalHitMock, @@ -16,6 +20,56 @@ import { import { GetMatchedThreats, ThreatListItem, ThreatMatchNamedQuery } from './types'; import { encodeThreatMatchNamedQuery } from './utils'; +describe('groupAndMergeSignalMatches', () => { + it('returns an empty array if there are no signals', () => { + expect(groupAndMergeSignalMatches([])).toEqual([]); + }); + + it('returns the same list if there are no duplicates', () => { + const signals = [getSignalHitMock({ _id: '1' }), getSignalHitMock({ _id: '2' })]; + const expectedSignals = [...signals]; + expect(groupAndMergeSignalMatches(signals)).toEqual(expectedSignals); + }); + + it('deduplicates signals with the same ID', () => { + const signals = [getSignalHitMock({ _id: '1' }), getSignalHitMock({ _id: '1' })]; + const expectedSignals = [signals[0]]; + expect(groupAndMergeSignalMatches(signals)).toEqual(expectedSignals); + }); + + it('merges the matched_queries of duplicate signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query3', 'query4'] }), + ]; + const [mergedSignal] = groupAndMergeSignalMatches(signals); + expect(mergedSignal.matched_queries).toEqual(['query1', 'query3', 'query4']); + }); + + it('does not deduplicate identical named queries on duplicate signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query1', 'query2'] }), + ]; + const [mergedSignal] = groupAndMergeSignalMatches(signals); + expect(mergedSignal.matched_queries).toEqual(['query1', 'query1', 'query2']); + }); + + it('merges the matched_queries of multiple signals', () => { + const signals = [ + getSignalHitMock({ _id: '1', matched_queries: ['query1'] }), + getSignalHitMock({ _id: '1', matched_queries: ['query3', 'query4'] }), + getSignalHitMock({ _id: '2', matched_queries: ['query1', 'query2'] }), + getSignalHitMock({ _id: '2', matched_queries: ['query5', 'query6'] }), + ]; + const mergedSignals = groupAndMergeSignalMatches(signals); + expect(mergedSignals.map((signal) => signal.matched_queries)).toEqual([ + ['query1', 'query3', 'query4'], + ['query1', 'query2', 'query5', 'query6'], + ]); + }); +}); + describe('buildMatchedIndicator', () => { let threats: ThreatListItem[]; let queries: ThreatMatchNamedQuery[]; @@ -96,7 +150,7 @@ describe('buildMatchedIndicator', () => { expect(indicators).toHaveLength(queries.length); }); - it('returns the indicator data the specified at threat.indicator by default', () => { + it('returns the indicator data specified at threat.indicator by default', () => { const indicators = buildMatchedIndicator({ queries, threats, @@ -260,13 +314,22 @@ describe('enrichSignalThreatMatches', () => { const indicators = get(enrichedHit._source, 'threat.indicator'); expect(indicators).toEqual([ - { existing: 'indicator' }, { domain: 'domain_1', matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, other: 'other_1', type: 'type_1', }, + { + domain: 'domain_2', + matched: { + atomic: 'domain_2', + field: 'threat.indicator.domain', + type: 'type_2', + }, + other: 'other_2', + type: 'type_2', + }, ]); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 8e5afb330b12c3..605594b4709837 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -6,7 +6,7 @@ import { get } from 'lodash'; -import type { SignalSearchResponse } from '../types'; +import type { SignalSearchResponse, SignalSourceHit } from '../types'; import type { GetMatchedThreats, ThreatIndicator, @@ -15,6 +15,28 @@ import type { } from './types'; import { extractNamedQueries } from './utils'; +const getSignalId = (signal: SignalSourceHit): string => signal._id; + +export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { + const dedupedHitsMap = signalHits.reduce>((acc, signalHit) => { + const signalId = getSignalId(signalHit); + if (!acc.has(signalId)) { + acc.set(signalId, signalHit); + } else { + const existingSignalHit = acc.get(signalId) as SignalSourceHit; + const existingQueries = existingSignalHit?.matched_queries ?? []; + const newQueries = signalHit.matched_queries ?? []; + existingSignalHit.matched_queries = [...existingQueries, ...newQueries]; + + acc.set(signalId, existingSignalHit); + } + + return acc; + }, new Map()); + const dedupedHits = Array.from(dedupedHitsMap.values()); + return dedupedHits; +}; + export const buildMatchedIndicator = ({ queries, threats, @@ -47,16 +69,16 @@ export const enrichSignalThreatMatches = async ( if (signalHits.length === 0) { return signals; } - // TODO: merge duplicate signals - const matches = signalHits.map((signalHit) => extractNamedQueries(signalHit)); - const matchedThreatIds = [...new Set(matches.flat().map(({ id }) => id))]; + const uniqueHits = groupAndMergeSignalMatches(signalHits); + const signalMatches = uniqueHits.map((signalHit) => extractNamedQueries(signalHit)); + const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))]; const matchedThreats = await getMatchedThreats(matchedThreatIds); - const matchedIndicators = matches.map((queries) => + const matchedIndicators = signalMatches.map((queries) => buildMatchedIndicator({ queries, threats: matchedThreats }) ); - const enrichedSignals = signalHits.map((signalHit, i) => { + const enrichedSignals = uniqueHits.map((signalHit, i) => { const threat = get(signalHit._source, 'threat') ?? {}; const existingIndicators = get(signalHit._source, 'threat.indicator') ?? []; From 0d256c63f7d3fcb921ca0df95746e756f6d406a9 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 21:22:16 -0600 Subject: [PATCH 14/40] Move default indicator path to constant --- .../signals/threat_mapping/enrich_signal_threat_matches.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 605594b4709837..6283123b0e13d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -15,6 +15,7 @@ import type { } from './types'; import { extractNamedQueries } from './utils'; +const DEFAULT_INDICATOR_PATH = 'threat.indicator'; const getSignalId = (signal: SignalSourceHit): string => signal._id; export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { @@ -40,7 +41,7 @@ export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): Signa export const buildMatchedIndicator = ({ queries, threats, - indicatorPath = 'threat.indicator', + indicatorPath = DEFAULT_INDICATOR_PATH, }: { queries: ThreatMatchNamedQuery[]; threats: ThreatListItem[]; From 2da0633373c28df294a8eb5928b9f66102fdb9f7 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 21:48:40 -0600 Subject: [PATCH 15/40] Testing some edge cases with signal enrichment --- .../enrich_signal_threat_matches.test.ts | 16 +++++++++++++++- .../enrich_signal_threat_matches.ts | 8 ++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index b8404a062d4cc5..720cb9d5bf1054 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -253,7 +253,7 @@ describe('enrichSignalThreatMatches', () => { ]); }); - it.skip('preserves an existing threat.indicator object on signals', async () => { + it('preserves an existing threat.indicator object on signals', async () => { const query = encodeThreatMatchNamedQuery( getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) ); @@ -277,6 +277,20 @@ describe('enrichSignalThreatMatches', () => { ]); }); + it('throws an error if threat is neither an object nor undefined', async () => { + const query = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) + ); + const signalHit = getSignalHitMock({ + _source: { '@timestamp': 'mocked', threat: 'whoops' }, + matched_queries: [query], + }); + const signals = getSignalsResponseMock([signalHit]); + await expect(() => enrichSignalThreatMatches(signals, getMatchedThreats)).rejects.toThrowError( + 'Expected threat field to be an object, but found: whoops' + ); + }); + it('merges duplicate matched signals into a single signal with multiple indicators', async () => { getMatchedThreats = async () => [ getThreatListItemMock({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 6283123b0e13d3..fa87aac8b2bacd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; +import { get, isObject } from 'lodash'; import type { SignalSearchResponse, SignalSourceHit } from '../types'; import type { @@ -81,7 +81,11 @@ export const enrichSignalThreatMatches = async ( const enrichedSignals = uniqueHits.map((signalHit, i) => { const threat = get(signalHit._source, 'threat') ?? {}; - const existingIndicators = get(signalHit._source, 'threat.indicator') ?? []; + if (!isObject(threat)) { + throw new Error(`Expected threat field to be an object, but found: ${threat}`); + } + const existingIndicatorValue = get(signalHit._source, 'threat.indicator') ?? []; + const existingIndicators = [existingIndicatorValue].flat(); // ensure indicators is an array return { ...signalHit, From e1206094dddea8d5d6acf3601d79515cd41276d2 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 22:41:00 -0600 Subject: [PATCH 16/40] Cover and test edge cases with threat enrichment generation --- .../enrich_signal_threat_matches.test.ts | 101 ++++++++++++++++++ .../enrich_signal_threat_matches.ts | 13 +-- 2 files changed, 108 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 720cb9d5bf1054..5d5d5ef5dbf02b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -206,6 +206,107 @@ describe('buildMatchedIndicator', () => { }, ]); }); + + it('returns only the match data if indicator field is absent', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: {}, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + matched: { + atomic: undefined, + field: 'threat.indicator.domain', + type: undefined, + }, + }, + ]); + }); + + it('returns only the match data if indicator field is an empty array', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { threat: { indicator: [] } }, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + matched: { + atomic: undefined, + field: 'threat.indicator.domain', + type: undefined, + }, + }, + ]); + }); + + it('returns data sans atomic from first indicator if indicator field is an array of objects', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: [ + { domain: 'foo', type: 'first' }, + { domain: 'bar', type: 'second' }, + ], + }, + }, + }), + ]; + + const indicators = buildMatchedIndicator({ + queries, + threats, + }); + + expect(indicators).toEqual([ + { + domain: 'foo', + matched: { + atomic: undefined, + field: 'threat.indicator.domain', + type: 'first', + }, + type: 'first', + }, + ]); + }); + + it('throws an error if indicator field is not an object or an array', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: 'not an object', + }, + }, + }), + ]; + + expect(() => + buildMatchedIndicator({ + queries, + threats, + }) + ).toThrowError('Expected indicator field to be an object, but found: not an object'); + }); }); describe('enrichSignalThreatMatches', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index fa87aac8b2bacd..bf45efbc063101 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -49,12 +49,13 @@ export const buildMatchedIndicator = ({ }): ThreatIndicator[] => queries.map((query) => { const matchedThreat = threats.find((threat) => threat._id === query.id); - // TODO what if this is an array? Grab the first? - const indicator = get(matchedThreat?._source, indicatorPath); - const atomic = get(matchedThreat?._source, query.value); - const type = get(indicator, 'type'); - // console.log('matchedThreat', JSON.stringify(matchedThreat, null, 2)); - // console.log('indicator', JSON.stringify(indicator, null, 2)); + const indicatorValue = get(matchedThreat?._source, indicatorPath) as unknown; + const indicator = [indicatorValue].flat()[0] ?? {}; + if (!isObject(indicator)) { + throw new Error(`Expected indicator field to be an object, but found: ${indicator}`); + } + const atomic = get(matchedThreat?._source, query.value) as unknown; + const type = get(indicator, 'type') as unknown; return { ...indicator, From fa1a8d20cf7238b1de6c11e3562f8c3505960d9b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 23:03:09 -0600 Subject: [PATCH 17/40] Fix logical error in TI enrichment We were previously adding the indicator's field to matched.field, instead of the corresponding event field that matched the indicator. In the normal case, the expectation is that the indicator field is self-evident, and thus we want to know the other side of the match on the event itself. Updates tests accordingly. --- .../enrich_signal_threat_matches.test.ts | 59 +++++++++---------- .../enrich_signal_threat_matches.ts | 2 +- .../tests/create_threat_matching.ts | 23 ++++---- 3 files changed, 39 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 5d5d5ef5dbf02b..c745becf183785 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -83,7 +83,9 @@ describe('buildMatchedIndicator', () => { }, }), ]; - queries = [getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' })]; + queries = [ + getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }), + ]; }); it('returns an empty list if queries is empty', () => { @@ -110,7 +112,7 @@ describe('buildMatchedIndicator', () => { threats, }); - expect(get(indicator, 'matched.field')).toEqual('threat.indicator.domain'); + expect(get(indicator, 'matched.field')).toEqual('event.field'); }); it('returns the type of the matched indicator as matched.type', () => { @@ -161,7 +163,7 @@ describe('buildMatchedIndicator', () => { domain: 'domain_1', matched: { atomic: 'domain_1', - field: 'threat.indicator.domain', + field: 'event.field', type: 'type_1', }, other: 'other_1', @@ -170,7 +172,7 @@ describe('buildMatchedIndicator', () => { ]); }); - it('returns the indicator data the specified at the custom path', () => { + it('returns the indicator data specified at the custom path', () => { threats = [ getThreatListItemMock({ _id: '123', @@ -199,7 +201,7 @@ describe('buildMatchedIndicator', () => { indicator_field: 'indicator_field_1', matched: { atomic: 'domain_1', - field: 'threat.indicator.domain', + field: 'event.field', type: 'indicator_type', }, type: 'indicator_type', @@ -224,7 +226,7 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, - field: 'threat.indicator.domain', + field: 'event.field', type: undefined, }, }, @@ -248,7 +250,7 @@ describe('buildMatchedIndicator', () => { { matched: { atomic: undefined, - field: 'threat.indicator.domain', + field: 'event.field', type: undefined, }, }, @@ -280,7 +282,7 @@ describe('buildMatchedIndicator', () => { domain: 'foo', matched: { atomic: undefined, - field: 'threat.indicator.domain', + field: 'event.field', type: 'first', }, type: 'first', @@ -311,6 +313,7 @@ describe('buildMatchedIndicator', () => { describe('enrichSignalThreatMatches', () => { let getMatchedThreats: GetMatchedThreats; + let matchedQuery: string; beforeEach(() => { getMatchedThreats = async () => [ @@ -321,6 +324,9 @@ describe('enrichSignalThreatMatches', () => { }, }), ]; + matchedQuery = encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '123', field: 'event.field', value: 'threat.indicator.domain' }) + ); }); it('performs no enrichment if there are no signals', async () => { @@ -331,12 +337,9 @@ describe('enrichSignalThreatMatches', () => { }); it('preserves existing threat.indicator objects on signals', async () => { - const query = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) - ); const signalHit = getSignalHitMock({ _source: { '@timestamp': 'mocked', threat: { indicator: [{ existing: 'indicator' }] } }, - matched_queries: [query], + matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); @@ -347,7 +350,7 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, other: 'other_1', type: 'type_1', }, @@ -355,12 +358,9 @@ describe('enrichSignalThreatMatches', () => { }); it('preserves an existing threat.indicator object on signals', async () => { - const query = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) - ); const signalHit = getSignalHitMock({ _source: { '@timestamp': 'mocked', threat: { indicator: { existing: 'indicator' } } }, - matched_queries: [query], + matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); @@ -371,7 +371,7 @@ describe('enrichSignalThreatMatches', () => { { existing: 'indicator' }, { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, other: 'other_1', type: 'type_1', }, @@ -379,12 +379,9 @@ describe('enrichSignalThreatMatches', () => { }); it('throws an error if threat is neither an object nor undefined', async () => { - const query = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) - ); const signalHit = getSignalHitMock({ _source: { '@timestamp': 'mocked', threat: 'whoops' }, - matched_queries: [query], + matched_queries: [matchedQuery], }); const signals = getSignalsResponseMock([signalHit]); await expect(() => enrichSignalThreatMatches(signals, getMatchedThreats)).rejects.toThrowError( @@ -407,19 +404,17 @@ describe('enrichSignalThreatMatches', () => { }, }), ]; - const query = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '123', value: 'threat.indicator.domain' }) - ); - const otherQuery = encodeThreatMatchNamedQuery( - getNamedQueryMock({ id: '456', value: 'threat.indicator.domain' }) - ); const signalHit = getSignalHitMock({ _id: 'signal123', - matched_queries: [query], + matched_queries: [matchedQuery], }); const otherSignalHit = getSignalHitMock({ _id: 'signal123', - matched_queries: [otherQuery], + matched_queries: [ + encodeThreatMatchNamedQuery( + getNamedQueryMock({ id: '456', field: 'event.other', value: 'threat.indicator.domain' }) + ), + ], }); const signals = getSignalsResponseMock([signalHit, otherSignalHit]); const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); @@ -431,7 +426,7 @@ describe('enrichSignalThreatMatches', () => { expect(indicators).toEqual([ { domain: 'domain_1', - matched: { atomic: 'domain_1', field: 'threat.indicator.domain', type: 'type_1' }, + matched: { atomic: 'domain_1', field: 'event.field', type: 'type_1' }, other: 'other_1', type: 'type_1', }, @@ -439,7 +434,7 @@ describe('enrichSignalThreatMatches', () => { domain: 'domain_2', matched: { atomic: 'domain_2', - field: 'threat.indicator.domain', + field: 'event.other', type: 'type_2', }, other: 'other_2', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index bf45efbc063101..58ba440324fbdd 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -59,7 +59,7 @@ export const buildMatchedIndicator = ({ return { ...indicator, - matched: { atomic, field: query.value, type }, + matched: { atomic, field: query.field, type }, }; }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 9b5e5b588ea281..3b7fbb45166e78 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -306,7 +306,7 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', - field: 'threat.indicator.domain', + field: 'destination.ip', type: 'url', }, provider: 'geenensp', @@ -326,7 +326,7 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', - field: 'threat.indicator.domain', + field: 'destination.ip', type: 'url', }, provider: 'geenensp', @@ -386,7 +386,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', - field: 'threat.indicator.ip', + field: 'source.ip', type: 'url', }, port: 57324, @@ -399,7 +399,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', - field: 'threat.indicator.ip', + field: 'source.ip', type: 'ip', }, provider: 'other_provider', @@ -467,7 +467,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', - field: 'threat.indicator.ip', + field: 'source.ip', type: 'url', }, port: 57324, @@ -480,7 +480,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', - field: 'threat.indicator.ip', + field: 'source.ip', type: 'ip', }, provider: 'other_provider', @@ -492,7 +492,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, - field: 'threat.indicator.port', + field: 'source.port', type: 'url', }, port: 57324, @@ -563,7 +563,7 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', - field: 'threat.indicator.domain', + field: 'destination.ip', type: 'url', }, provider: 'geenensp', @@ -583,7 +583,7 @@ export default ({ getService }: FtrProviderContext) => { first_seen: '2021-01-26T11:09:04.000Z', matched: { atomic: '159.89.119.67', - field: 'threat.indicator.domain', + field: 'destination.ip', type: 'url', }, provider: 'geenensp', @@ -599,7 +599,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: '45.115.45.3', - field: 'threat.indicator.ip', + field: 'source.ip', type: 'url', }, port: 57324, @@ -612,7 +612,7 @@ export default ({ getService }: FtrProviderContext) => { ip: '45.115.45.3', matched: { atomic: 57324, - field: 'threat.indicator.port', + field: 'source.port', type: 'url', }, port: 57324, @@ -623,7 +623,6 @@ export default ({ getService }: FtrProviderContext) => { }, ]); }); - it('deduplicates a signal if it is found in multiple separate query loops'); }); }); }); From aaec43254cd08c6bbb6c1e716d044ce0c5d02c55 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 23:22:37 -0600 Subject: [PATCH 18/40] Document behavior when an indicator matched but is absent on enrichment This could occur if the indicator index is updated while a rule is being run. --- .../enrich_signal_threat_matches.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index c745becf183785..c42f7695160677 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -357,6 +357,24 @@ describe('enrichSignalThreatMatches', () => { ]); }); + // TODO note in PR + it('provides only match data if the matched threat cannot be found', async () => { + getMatchedThreats = async () => []; + const signalHit = getSignalHitMock({ + matched_queries: [matchedQuery], + }); + const signals = getSignalsResponseMock([signalHit]); + const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + const [enrichedHit] = enrichedSignals.hits.hits; + const indicators = get(enrichedHit._source, 'threat.indicator'); + + expect(indicators).toEqual([ + { + matched: { atomic: undefined, field: 'event.field', type: undefined }, + }, + ]); + }); + it('preserves an existing threat.indicator object on signals', async () => { const signalHit = getSignalHitMock({ _source: { '@timestamp': 'mocked', threat: { indicator: { existing: 'indicator' } } }, From 3e31d6a4f7a64aa9ab1d7f4e49aca750e286623d Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 23:24:02 -0600 Subject: [PATCH 19/40] Add followup note --- .../signals/threat_mapping/enrich_signal_threat_matches.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 58ba440324fbdd..10ad9e6aa4191c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -72,6 +72,7 @@ export const enrichSignalThreatMatches = async ( return signals; } + // TODO update hits total to account for deduping const uniqueHits = groupAndMergeSignalMatches(signalHits); const signalMatches = uniqueHits.map((signalHit) => extractNamedQueries(signalHit)); const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))]; From 537fa3cbbfbaed13c0a3f0365dc42c28b30047e8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 2 Feb 2021 23:45:40 -0600 Subject: [PATCH 20/40] Add basic unit test for our enrichment function This just verifies that the enrichment function gets invoked with search results. --- .../signals/search_after_bulk_create.test.ts | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 418d30711169e3..bf2b33f21f0dcf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -14,6 +14,7 @@ import { repeatedSearchResultsWithSortId, repeatedSearchResultsWithNoSortId, sampleDocSearchResultsNoSortIdNoHits, + sampleDocWithSortId, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; import { buildRuleMessageFactory } from './rule_messages'; @@ -870,4 +871,93 @@ describe('searchAfterAndBulkCreate', () => { expect(createdSignalsCount).toEqual(4); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); }); + + it('invokes the enrichment callback with signal search results', async () => { + const sampleParams = sampleRuleAlertParams(30); + mockService.callCluster + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9))) + .mockResolvedValueOnce({ + took: 100, + errors: false, + items: [ + { + create: { + status: 201, + }, + }, + ], + }) + .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits()); + + const mockEnrichment = jest.fn((a) => a); + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ + enrichment: mockEnrichment, + ruleParams: sampleParams, + gap: moment.duration(2, 'm'), + previousStartedAt: moment().subtract(10, 'm').toDate(), + listClient, + exceptionsList: [], + services: mockService, + logger: mockLogger, + eventsTelemetry: undefined, + id: sampleRuleGuid, + inputIndexPattern, + signalsIndex: DEFAULT_SIGNALS_INDEX, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + pageSize: 1, + filter: undefined, + refresh: false, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + buildRuleMessage, + }); + + expect(mockEnrichment).toHaveBeenCalledWith( + expect.objectContaining({ + hits: expect.objectContaining({ + hits: expect.arrayContaining([ + expect.objectContaining({ + ...sampleDocWithSortId(), + _id: expect.any(String), + }), + ]), + }), + }) + ); + expect(success).toEqual(true); + expect(mockService.callCluster).toHaveBeenCalledTimes(7); + expect(createdSignalsCount).toEqual(3); + expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); + }); }); From 12a0c87c50cfe1e277019ba7c371022fb9c674a2 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 3 Feb 2021 23:12:10 -0600 Subject: [PATCH 21/40] Update license headers for new files --- .../threat_mapping/enrich_signal_threat_matches.mock.ts | 5 +++-- .../threat_mapping/enrich_signal_threat_matches.test.ts | 5 +++-- .../signals/threat_mapping/enrich_signal_threat_matches.ts | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts index d694ba3e532b0c..a3ff932e97886e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.mock.ts @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ import { SignalSearchResponse, SignalSourceHit } from '../types'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index c42f7695160677..35932c8c23cfe8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ import { get } from 'lodash'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 10ad9e6aa4191c..54511fe8180836 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ import { get, isObject } from 'lodash'; From 5675a786875c1a057b02e7c26011bb79ef253f59 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 22:04:21 -0600 Subject: [PATCH 22/40] Remove unused threatintel archive I made both of these before we were clear on the direction we were taking here. --- .../filebeat/threat_intel_nested/data.json | 139 ---------- .../threat_intel_nested/mappings.json | 244 ------------------ 2 files changed, 383 deletions(-) delete mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json delete mode 100644 x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json deleted file mode 100644 index 127d4bb750e771..00000000000000 --- a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/data.json +++ /dev/null @@ -1,139 +0,0 @@ -{ - "type": "doc", - "value": { - "id": "978783", - "index": "filebeat-8.0.0-2021.01.26-000001", - "source": { - "@timestamp": "2021-01-26T11:09:05.529Z", - "agent": { - "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", - "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", - "name": "MacBook-Pro-de-Gloria.local", - "type": "filebeat", - "version": "8.0.0" - }, - "ecs": { - "version": "1.6.0" - }, - "event": { - "category": "threat", - "created": "2021-01-26T11:09:05.529Z", - "dataset": "threatintel.abuseurl", - "ingested": "2021-01-26T11:09:06.595350Z", - "kind": "enrichment", - "module": "threatintel", - "reference": "https://urlhaus.abuse.ch/url/978783/", - "type": "indicator" - }, - "fileset": { - "name": "abuseurl" - }, - "input": { - "type": "httpjson" - }, - "service": { - "type": "threatintel" - }, - "tags": [ - "threatintel-abuseurls", - "forwarded" - ], - "threat": { - "indicator": { - "domain": "159.89.119.67", - "first_seen": "2021-01-26T11:09:04.000Z", - "provider": "geenensp", - "type": "url", - "url": { - "full": "http://159.89.119.67:59600/bin.sh", - "scheme": "http" - } - } - }, - "threatintel": { - "abuseurl": { - "blacklists": { - "spamhaus_dbl": "not listed", - "surbl": "not listed" - }, - "larted": false, - "tags": null, - "threat": "malware_download", - "url_status": "online" - } - } - } - } -} - -{ - "type": "doc", - "value": { - "id": "978782", - "index": "filebeat-8.0.0-2021.01.26-000001", - "source": { - "@timestamp": "2021-01-26T11:09:05.529Z", - "agent": { - "ephemeral_id": "b7b56c3e-1f27-4c69-96f4-aa9ca47888d0", - "id": "69acb5f0-1e79-4cfe-a4dc-e0dbf229ff51", - "name": "MacBook-Pro-de-Gloria.local", - "type": "filebeat", - "version": "8.0.0" - }, - "ecs": { - "version": "1.6.0" - }, - "event": { - "category": "threat", - "created": "2021-01-26T11:09:05.529Z", - "dataset": "threatintel.abuseurl", - "ingested": "2021-01-26T11:09:06.616763Z", - "kind": "enrichment", - "module": "threatintel", - "reference": "https://urlhaus.abuse.ch/url/978782/", - "type": "indicator" - }, - "fileset": { - "name": "abuseurl" - }, - "input": { - "type": "httpjson" - }, - "service": { - "type": "threatintel" - }, - "tags": [ - "threatintel-abuseurls", - "forwarded" - ], - "threat": { - "indicator": { - "domain": "125.46.136.106", - "first_seen": "2021-01-26T11:06:03.000Z", - "provider": "geenensp", - "type": "url", - "url": { - "full": "http://125.46.136.106:52014/bin.sh", - "scheme": "http" - } - } - }, - "threatintel": { - "abuseurl": { - "blacklists": { - "spamhaus_dbl": "not listed", - "surbl": "not listed" - }, - "larted": true, - "tags": [ - "32-bit", - "elf", - "mips" - ], - "threat": "malware_download", - "url_status": "online" - } - } - } - } -} diff --git a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json b/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json deleted file mode 100644 index 73f9b5ba1934f7..00000000000000 --- a/x-pack/test/functional/es_archives/filebeat/threat_intel_nested/mappings.json +++ /dev/null @@ -1,244 +0,0 @@ -{ - "type": "index", - "value": { - "aliases": {}, - "index": "filebeat-8.0.0-2021.01.26-000001", - "mappings": { - "_meta": { - "beat": "filebeat", - "version": "7.0.0" - }, - "properties": { - "@timestamp": { - "type": "date" - }, - "@version": { - "ignore_above": 1024, - "type": "keyword" - }, - "threat": { - "properties": { - "framework": { - "ignore_above": 1024, - "type": "keyword" - }, - "indicator": { - "type": "nested", - "properties": { - "as": { - "properties": { - "number": { - "type": "long" - }, - "organization": { - "properties": { - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - }, - "confidence": { - "ignore_above": 1024, - "type": "keyword" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" - }, - "description": { - "type": "wildcard" - }, - "domain": { - "ignore_above": 1024, - "type": "keyword" - }, - "email": { - "properties": { - "address": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "first_seen": { - "type": "date" - }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "continent_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "country_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "location": { - "type": "geo_point" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_iso_code": { - "ignore_above": 1024, - "type": "keyword" - }, - "region_name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "last_seen": { - "type": "date" - }, - "marking": { - "properties": { - "tlp": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "matched": { - "properties": { - "atomic": { - "ignore_above": 1024, - "type": "keyword" - }, - "field": { - "ignore_above": 1024, - "type": "keyword" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "module": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "provider": { - "ignore_above": 1024, - "type": "keyword" - }, - "scanner_stats": { - "type": "long" - }, - "sightings": { - "type": "long" - }, - "type": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "tactic": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "technique": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - }, - "subtechnique": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - } - } - } - } - } - } - } - } - }, - "settings": { - "index": { - "lifecycle": { - "name": "filebeat-8.0.0", - "rollover_alias": "filebeat-filebeat-8.0.0" - }, - "mapping": { - "total_fields": { - "limit": "10000" - } - }, - "number_of_replicas": "0", - "number_of_shards": "1", - "refresh_interval": "5s" - } - } - } -} From 9481f81fbbf37db793d1df7480c3cddbcb06ee1b Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 22:05:56 -0600 Subject: [PATCH 23/40] Bump signals version to allows some updates in patch releases --- .../lib/detection_engine/routes/index/get_signals_template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index 83992f2b3f1c3f..25d208874e2064 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -8,7 +8,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 15; +export const SIGNALS_TEMPLATE_VERSION = 24; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { From fc03ba8e37fe755a28090367029b528a8abc6b1a Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 22:52:43 -0600 Subject: [PATCH 24/40] Fix typings of threat list item We were conflating the type of the underlying document with the type of the search response for that document. This is now addressed with two types: ThreatListDoc and ThreatListItem, respectively. ThreatListDoc isn't the most distinguishing name but it avoids a lot of unnecessary renaming for the existing concept of ThreatListItem. --- .../build_threat_mapping_filter.mock.ts | 44 +++++++++---------- .../build_threat_mapping_filter.test.ts | 23 ++++++---- .../signals/threat_mapping/types.ts | 10 +++-- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index 87a5f8143dfe8e..6c15b6e0afda7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -9,7 +9,7 @@ import { ThreatMapping } from '../../../../../common/detection_engine/schemas/ty import { Filter } from 'src/plugins/data/common'; import { SearchResponse } from 'elasticsearch'; -import { ThreatListItem } from './types'; +import { ThreatListDoc, ThreatListItem } from './types'; export const getThreatMappingMock = (): ThreatMapping => { return [ @@ -62,7 +62,7 @@ export const getThreatMappingMock = (): ThreatMapping => { ]; }; -export const getThreatListSearchResponseMock = (): SearchResponse => ({ +export const getThreatListSearchResponseMock = (): SearchResponse => ({ took: 0, timed_out: false, _shards: { @@ -74,31 +74,29 @@ export const getThreatListSearchResponseMock = (): SearchResponse = {}): ThreatListItem => ({ - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', - ip: '192.168.0.0.1', - }, - source: { - ip: '127.0.0.1', - port: 1, - }, - destination: { - ip: '127.0.0.1', - port: 1, + _id: '123', + _index: 'threat_index', + _type: '_doc', + _score: 0, + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + ip: '192.168.0.0.1', + }, + source: { + ip: '127.0.0.1', + port: 1, + }, + destination: { + ip: '127.0.0.1', + port: 1, + }, }, ...overrides, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts index 792fa889e395d8..c499b337162b2f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.test.ts @@ -132,13 +132,15 @@ describe('build_threat_mapping_filter', () => { ], }, ], - threatListItem: { - '@timestamp': '2020-09-09T21:59:13Z', - host: { - name: 'host-1', - // since ip is missing this entire AND clause should be dropped + threatListItem: getThreatListItemMock({ + _source: { + '@timestamp': '2020-09-09T21:59:13Z', + host: { + name: 'host-1', + // since ip is missing this entire AND clause should be dropped + }, }, - }, + }), }); expect(item).toEqual([]); }); @@ -170,14 +172,14 @@ describe('build_threat_mapping_filter', () => { ], }, ], - threatListItem: { + threatListItem: getThreatListItemMock({ _source: { '@timestamp': '2020-09-09T21:59:13Z', host: { name: 'host-1', }, }, - }, + }), }); expect(item).toEqual([ { @@ -315,7 +317,10 @@ describe('build_threat_mapping_filter', () => { test('it should return an empty boolean clause given an empty object for a threat list item', () => { const threatMapping = getThreatMappingMock(); - const innerClause = createAndOrClauses({ threatMapping, threatListItem: { _source: {} } }); + const innerClause = createAndOrClauses({ + threatMapping, + threatListItem: getThreatListItemMock({ _source: {} }), + }); expect(innerClause).toEqual({ bool: { minimum_should_match: 1, should: [] } }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 44cf18ab75d704..98878b0572d765 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -5,7 +5,9 @@ * 2.0. */ +import { SearchResponse } from 'elasticsearch'; import { Duration } from 'moment'; + import { ListClient } from '../../../../../../lists/server'; import { Type, @@ -178,13 +180,15 @@ export interface GetSortWithTieBreakerOptions { listItemIndex: string; } +export interface ThreatListDoc { + [key: string]: unknown; +} + /** * This is an ECS document being returned, but the user could return or use non-ecs based * documents potentially. */ -export interface ThreatListItem { - [key: string]: unknown; -} +export type ThreatListItem = SearchResponse['hits']['hits'][number]; export interface ThreatIndicator { [key: string]: unknown; From 09cca36358d1e0ad07ea6544a8784844bbdeb79a Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 23:09:04 -0600 Subject: [PATCH 25/40] Update test mock to be aware of (but not care about) named queries --- .../build_threat_mapping_filter.mock.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts index 6c15b6e0afda7b..00622a585550a4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_mapping_filter.mock.ts @@ -176,13 +176,17 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'host.name': 'host-1' } }], + should: [ + { match: { 'host.name': { query: 'host-1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, { bool: { - should: [{ match: { 'host.ip': '192.168.0.0.1' } }], + should: [ + { match: { 'host.ip': { query: '192.168.0.0.1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, @@ -194,13 +198,19 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'destination.ip': '127.0.0.1' } }], + should: [ + { + match: { 'destination.ip': { query: '127.0.0.1', _name: expect.any(String) } }, + }, + ], minimum_should_match: 1, }, }, { bool: { - should: [{ match: { 'destination.port': port } }], + should: [ + { match: { 'destination.port': { query: port, _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, @@ -212,7 +222,7 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'source.port': port } }], + should: [{ match: { 'source.port': { query: port, _name: expect.any(String) } } }], minimum_should_match: 1, }, }, @@ -224,7 +234,9 @@ export const getThreatMappingFilterShouldMock = (port = 1) => ({ filter: [ { bool: { - should: [{ match: { 'source.ip': '127.0.0.1' } }], + should: [ + { match: { 'source.ip': { query: '127.0.0.1', _name: expect.any(String) } } }, + ], minimum_should_match: 1, }, }, From 679d138dae74e1aab2ad52618cb65385225ef3c1 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 23:48:18 -0600 Subject: [PATCH 26/40] Remove/update outdated comments This code was modified to perform two searches instead of one; at that time, a lot of this code was duplicated and modified slightly, and these misleading comments were a result. I removed the ones that were no longer relevant, but left a TODO for one that could be a bug. --- .../detection_engine/signals/search_after_bulk_create.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index f78c9e20251568..75e7042dd74259 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -119,17 +119,12 @@ export const searchAfterAndBulkCreate = async ({ backupSortId = lastSortId[0]; hasBackupSortId = true; } else { - // TODO: This comment does not seem to match the code; this is the first loop and the first search, so how could the initial search result be known? - // answer: these were previously not blocking searches, and so the "second" search actually executed first TLDR outdated comment - // if no sort id on backup search and the initial search result was also empty logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB')); hasBackupSortId = false; } mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]); - // TODO again, on the first loop this is the FIRST search and toReturn is default values - // merge the search result from the secondary search with the first toReturn = mergeReturns([ toReturn, createSearchAfterReturnTypeFromResponse({ @@ -170,7 +165,8 @@ export const searchAfterAndBulkCreate = async ({ }), ]); - // TODO it's unclear why hits are guaranteed here; the type appears to be identical to the previous result which has more guards + // TODO this comment was written when this code lived in a different + // TODO location. We should verify whether this is still true // we are guaranteed to have searchResult hits at this point // because we check before if the totalHits or // searchResult.hits.hits.length is 0 From e286aa6b0b18670823bbac6e4db9bcfaf68c0701 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 4 Feb 2021 23:50:04 -0600 Subject: [PATCH 27/40] Remove outdated comment Documents will always have _id. --- .../signals/threat_mapping/create_threat_signals.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 9b29efbfd6b877..b8af101d2ec04c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -92,7 +92,6 @@ export const createThreatSignals = async ({ }); const getMatchedThreats: GetMatchedThreats = async (ids) => { - // TODO should _id be a configurable field? const matchedThreatsFilter = { query: { bool: { From 52926af43e601bb830f40bf342a8abb7c1d349b4 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 5 Feb 2021 15:56:26 -0600 Subject: [PATCH 28/40] Update enriched signals' total to account for deduplication If a given signal matched on multiple indicators in different loops of our indicator query, it may appear multiple times. Our enrichment performs the merging of those duplicated results, but did not previously update the response's total field to account for this. I don't believe that anything downstream is actually using this field and that we are instead operating on the length of hits and the response from the bulk create request, but this keeps things consistent in case that changes. --- .../threat_mapping/enrich_signal_threat_matches.test.ts | 1 + .../threat_mapping/enrich_signal_threat_matches.ts | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 35932c8c23cfe8..eb6f7e60bbc95f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -437,6 +437,7 @@ describe('enrichSignalThreatMatches', () => { }); const signals = getSignalsResponseMock([signalHit, otherSignalHit]); const enrichedSignals = await enrichSignalThreatMatches(signals, getMatchedThreats); + expect(enrichedSignals.hits.total).toEqual(expect.objectContaining({ value: 1 })); expect(enrichedSignals.hits.hits).toHaveLength(1); const [enrichedHit] = enrichedSignals.hits.hits; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 54511fe8180836..67f39990825864 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -73,7 +73,6 @@ export const enrichSignalThreatMatches = async ( return signals; } - // TODO update hits total to account for deduping const uniqueHits = groupAndMergeSignalMatches(signalHits); const signalMatches = uniqueHits.map((signalHit) => extractNamedQueries(signalHit)); const matchedThreatIds = [...new Set(signalMatches.flat().map(({ id }) => id))]; @@ -101,8 +100,14 @@ export const enrichSignalThreatMatches = async ( }, }; }); - // eslint-disable-next-line require-atomic-updates + /* eslint-disable require-atomic-updates */ signals.hits.hits = enrichedSignals; + if (isObject(signals.hits.total)) { + signals.hits.total.value = enrichedSignals.length; + } else { + signals.hits.total = enrichedSignals.length; + } + /* eslint-enable require-atomic-updates */ return signals; }; From ce60822e6f35fde23e426b3ce5f42ea64065a2eb Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Fri, 5 Feb 2021 16:00:34 -0600 Subject: [PATCH 29/40] Remove development comments --- .../threat_mapping/enrich_signal_threat_matches.test.ts | 1 - .../security_and_spaces/tests/create_threat_matching.ts | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index eb6f7e60bbc95f..17be7e17e86f8d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -358,7 +358,6 @@ describe('enrichSignalThreatMatches', () => { ]); }); - // TODO note in PR it('provides only match data if the matched threat cannot be found', async () => { getMatchedThreats = async () => []; const signalHit = getSignalHitMock({ diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 3b7fbb45166e78..680b879fa4acb3 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -455,9 +455,7 @@ export default ({ getService }: FtrProviderContext) => { const { hits } = signalsOpen.hits; const threats = hits.map((hit) => hit._source.threat); - // TODO how should a signal be enriched if a single indicator matches - // on multiple fields? As evidenced below, we currently duplicate the - // indicator with a different matched object. + expect(threats).to.eql([ { indicator: [ From a96c773977fbbb5780cb14c2d7cb2f5b89e39fa8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Mon, 8 Feb 2021 16:50:58 -0600 Subject: [PATCH 30/40] Add JSDoc for our special template version constant --- .../routes/index/get_signals_template.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index 25d208874e2064..48036ec73511b5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -8,6 +8,19 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; +/** + @constant + @type {number} + @description This value represents the template version assumed by app code. + If this number is greater than the user's signals index version, the + detections UI will attempt to update the signals template and roll over to + a new signals index. + + If making mappings changes in a patch release, this number should be incremented by 1. + If making mappings changes in a minor release, this number should be + incremented by 10 in order to add "room" for the aforementioned patch + release +*/ export const SIGNALS_TEMPLATE_VERSION = 24; export const MIN_EQL_RULE_INDEX_VERSION = 2; From a99978a3f28811153336df30f9ba6ebaaec04fee Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:17:30 -0600 Subject: [PATCH 31/40] Remove outdated comments --- .../detection_engine/signals/search_after_bulk_create.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index 75e7042dd74259..061aa4bba5a412 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -165,12 +165,6 @@ export const searchAfterAndBulkCreate = async ({ }), ]); - // TODO this comment was written when this code lived in a different - // TODO location. We should verify whether this is still true - // we are guaranteed to have searchResult hits at this point - // because we check before if the totalHits or - // searchResult.hits.hits.length is 0 - // call this function setSortIdOrExit() const lastSortId = searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort; if (lastSortId != null && lastSortId.length !== 0) { sortId = lastSortId[0]; From 863e54603ea1c7fd5a14f3f97d47908795b35b53 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:17:49 -0600 Subject: [PATCH 32/40] Add an additional test permutation for error cases Ensure that we throw an error if the indicator field is either a primitive or an array of primitives. --- .../enrich_signal_threat_matches.test.ts | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts index 17be7e17e86f8d..3c0765b56ae20b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.test.ts @@ -291,7 +291,7 @@ describe('buildMatchedIndicator', () => { ]); }); - it('throws an error if indicator field is not an object or an array', () => { + it('throws an error if indicator field is a not an object', () => { threats = [ getThreatListItemMock({ _id: '123', @@ -310,6 +310,26 @@ describe('buildMatchedIndicator', () => { }) ).toThrowError('Expected indicator field to be an object, but found: not an object'); }); + + it('throws an error if indicator field is not an array of objects', () => { + threats = [ + getThreatListItemMock({ + _id: '123', + _source: { + threat: { + indicator: ['not an object'], + }, + }, + }), + ]; + + expect(() => + buildMatchedIndicator({ + queries, + threats, + }) + ).toThrowError('Expected indicator field to be an object, but found: not an object'); + }); }); describe('enrichSignalThreatMatches', () => { From 767739fdfc951ea3d854fad386800e409431ee22 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:20:10 -0600 Subject: [PATCH 33/40] Remove unnecessary coalescing These values are already defaulted in the parent, and the types are correct in that these cannot be undefined. --- .../signals/threat_mapping/create_threat_signals.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index b8af101d2ec04c..9c985457ed06b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -103,8 +103,8 @@ export const createThreatSignals = async ({ }; const threatResponse = await getThreatList({ callCluster: services.callCluster, - exceptionItems: exceptionItems ?? [], - threatFilters: [...(threatFilters ?? []), matchedThreatsFilter], + exceptionItems, + threatFilters: [...threatFilters, matchedThreatsFilter], query: threatQuery, language: threatLanguage, index: threatIndex, @@ -117,7 +117,7 @@ export const createThreatSignals = async ({ perPage: undefined, }); - return threatResponse.hits.hits ?? []; + return threatResponse.hits.hits; }; const threatEnrichment = (signals: SignalSearchResponse): Promise => From bd2a4672d6f1b309670d19275f73d04f47562ce3 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:38:42 -0600 Subject: [PATCH 34/40] Move logic to build threat enrichment function into helper --- .../threat_mapping/build_threat_enrichment.ts | 55 +++++++++++++++++++ .../threat_mapping/create_threat_signals.ts | 43 ++++----------- .../signals/threat_mapping/types.ts | 12 ++++ 3 files changed, 79 insertions(+), 31 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts new file mode 100644 index 00000000000000..b14d1482189388 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/build_threat_enrichment.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { SignalSearchResponse, SignalsEnrichment } from '../types'; +import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; +import { getThreatList } from './get_threat_list'; +import { BuildThreatEnrichmentOptions, GetMatchedThreats } from './types'; + +export const buildThreatEnrichment = ({ + buildRuleMessage, + exceptionItems, + listClient, + logger, + services, + threatFilters, + threatIndex, + threatLanguage, + threatQuery, +}: BuildThreatEnrichmentOptions): SignalsEnrichment => { + const getMatchedThreats: GetMatchedThreats = async (ids) => { + const matchedThreatsFilter = { + query: { + bool: { + filter: { + ids: { values: ids }, + }, + }, + }, + }; + const threatResponse = await getThreatList({ + callCluster: services.callCluster, + exceptionItems, + threatFilters: [...threatFilters, matchedThreatsFilter], + query: threatQuery, + language: threatLanguage, + index: threatIndex, + listClient, + searchAfter: undefined, + sortField: undefined, + sortOrder: undefined, + logger, + buildRuleMessage, + perPage: undefined, + }); + + return threatResponse.hits.hits; + }; + + return (signals: SignalSearchResponse): Promise => + enrichSignalThreatMatches(signals, getMatchedThreats); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 9c985457ed06b9..475c2e8c9c0326 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -13,6 +13,7 @@ import { createThreatSignal } from './create_threat_signal'; import { SearchAfterAndBulkCreateReturnType, SignalSearchResponse } from '../types'; import { combineConcurrentResults } from './utils'; import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; +import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignals = async ({ threatMapping, @@ -91,37 +92,17 @@ export const createThreatSignals = async ({ perPage, }); - const getMatchedThreats: GetMatchedThreats = async (ids) => { - const matchedThreatsFilter = { - query: { - bool: { - filter: { - ids: { values: ids }, - }, - }, - }, - }; - const threatResponse = await getThreatList({ - callCluster: services.callCluster, - exceptionItems, - threatFilters: [...threatFilters, matchedThreatsFilter], - query: threatQuery, - language: threatLanguage, - index: threatIndex, - listClient, - searchAfter: undefined, - sortField: undefined, - sortOrder: undefined, - logger, - buildRuleMessage, - perPage: undefined, - }); - - return threatResponse.hits.hits; - }; - - const threatEnrichment = (signals: SignalSearchResponse): Promise => - enrichSignalThreatMatches(signals, getMatchedThreats); + const threatEnrichment = buildThreatEnrichment({ + buildRuleMessage, + exceptionItems, + listClient, + logger, + services, + threatFilters, + threatIndex, + threatLanguage, + threatQuery, + }); while (threatList.hits.hits.length !== 0) { const chunks = chunk(itemsPerSearch, threatList.hits.hits); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 98878b0572d765..b80d3faf9b61c7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -205,3 +205,15 @@ export interface ThreatMatchNamedQuery { } export type GetMatchedThreats = (ids: string[]) => Promise; + +export interface BuildThreatEnrichmentOptions { + buildRuleMessage: BuildRuleMessage; + exceptionItems: ExceptionListItemSchema[]; + listClient: ListClient; + logger: Logger; + services: AlertServices; + threatFilters: PartialFilter[]; + threatIndex: ThreatIndex; + threatLanguage: ThreatLanguageOrUndefined; + threatQuery: ThreatQuery; +} From b6e176ced660a38e24db45a7303681d72b92f027 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:53:28 -0600 Subject: [PATCH 35/40] Refactor code to allow typescript to infer our type narrowing existingSignalHit could not be undefined on line 30 here, but typescript could not infer this from the !acc.has() call. --- .../signals/threat_mapping/enrich_signal_threat_matches.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 67f39990825864..24307248375a83 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -22,10 +22,11 @@ const getSignalId = (signal: SignalSourceHit): string => signal._id; export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { const dedupedHitsMap = signalHits.reduce>((acc, signalHit) => { const signalId = getSignalId(signalHit); - if (!acc.has(signalId)) { + const existingSignalHit = acc.get(signalId); + + if (existingSignalHit == null) { acc.set(signalId, signalHit); } else { - const existingSignalHit = acc.get(signalId) as SignalSourceHit; const existingQueries = existingSignalHit?.matched_queries ?? []; const newQueries = signalHit.matched_queries ?? []; existingSignalHit.matched_queries = [...existingQueries, ...newQueries]; From eca579b19849ab688330c5a0944eb909ceb1fcfd Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:55:12 -0600 Subject: [PATCH 36/40] Use a POJO over a Map We were using a map previously in order to use .has() for a predicate, but code has since been refactored to make that unnecessary. --- .../threat_mapping/enrich_signal_threat_matches.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 24307248375a83..8a292f92480faa 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -20,23 +20,23 @@ const DEFAULT_INDICATOR_PATH = 'threat.indicator'; const getSignalId = (signal: SignalSourceHit): string => signal._id; export const groupAndMergeSignalMatches = (signalHits: SignalSourceHit[]): SignalSourceHit[] => { - const dedupedHitsMap = signalHits.reduce>((acc, signalHit) => { + const dedupedHitsMap = signalHits.reduce>((acc, signalHit) => { const signalId = getSignalId(signalHit); - const existingSignalHit = acc.get(signalId); + const existingSignalHit = acc[signalId]; if (existingSignalHit == null) { - acc.set(signalId, signalHit); + acc[signalId] = signalHit; } else { const existingQueries = existingSignalHit?.matched_queries ?? []; const newQueries = signalHit.matched_queries ?? []; existingSignalHit.matched_queries = [...existingQueries, ...newQueries]; - acc.set(signalId, existingSignalHit); + acc[signalId] = existingSignalHit; } return acc; - }, new Map()); - const dedupedHits = Array.from(dedupedHitsMap.values()); + }, {}); + const dedupedHits = Object.values(dedupedHitsMap); return dedupedHits; }; From 9b8e343e1b9b5b13a30fb916bfc0d642e99fccc8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 15:59:50 -0600 Subject: [PATCH 37/40] Explicitly type our enriched signals These are being typed implicitly and verified against SignalSourceHit[] on the assignment below, but this makes the types explicit and surfaces a type error here instead of the subsequent assignment. --- .../signals/threat_mapping/enrich_signal_threat_matches.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts index 8a292f92480faa..c298ef98ebcd53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/enrich_signal_threat_matches.ts @@ -82,7 +82,7 @@ export const enrichSignalThreatMatches = async ( buildMatchedIndicator({ queries, threats: matchedThreats }) ); - const enrichedSignals = uniqueHits.map((signalHit, i) => { + const enrichedSignals: SignalSourceHit[] = uniqueHits.map((signalHit, i) => { const threat = get(signalHit._source, 'threat') ?? {}; if (!isObject(threat)) { throw new Error(`Expected threat field to be an object, but found: ${threat}`); From 0d8223f689bfe0e4b024769c94a149ad2f325010 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 16:04:43 -0600 Subject: [PATCH 38/40] Add an explanatory note about these test results --- .../security_and_spaces/tests/create_threat_matching.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts index 680b879fa4acb3..9e1c290d160590 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_threat_matching.ts @@ -484,6 +484,11 @@ export default ({ getService }: FtrProviderContext) => { provider: 'other_provider', type: 'ip', }, + // We do not merge matched indicators during enrichment, so in + // certain circumstances a given indicator document could appear + // multiple times in an enriched alert (albeit with different + // threat.indicator.matched data). That's the case with the + // first and third indicators matched, here. { description: 'this should match auditbeat/hosts on both port and ip', first_seen: '2021-01-26T11:06:03.000Z', From 1b973fc9461573211125e0b30e4376595e1d5490 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Wed, 10 Feb 2021 19:26:04 -0600 Subject: [PATCH 39/40] Remove unused imports These references were moved into buildThreatEnrichment --- .../signals/threat_mapping/create_threat_signals.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 475c2e8c9c0326..7690eb5eb1d554 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -8,11 +8,10 @@ import chunk from 'lodash/fp/chunk'; import { getThreatList, getThreatListCount } from './get_threat_list'; -import { CreateThreatSignalsOptions, GetMatchedThreats } from './types'; +import { CreateThreatSignalsOptions } from './types'; import { createThreatSignal } from './create_threat_signal'; -import { SearchAfterAndBulkCreateReturnType, SignalSearchResponse } from '../types'; +import { SearchAfterAndBulkCreateReturnType } from '../types'; import { combineConcurrentResults } from './utils'; -import { enrichSignalThreatMatches } from './enrich_signal_threat_matches'; import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignals = async ({ From bf35b55fe1592994d2ec232ee2c02b6103de3fa8 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Thu, 11 Feb 2021 16:36:49 -0600 Subject: [PATCH 40/40] Remove threat mappings accidentally brought in with indicator work I copied the entirety of the `threat` mappings in order to get the `threat.indicator` ones, but it looks like these were added at some point too. I'd rather these not be added incidentally. If we need them, we should do so explicitly. --- .../routes/index/ecs_mapping.json | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json index b4e6e4d1400c50..70b62d569b9d37 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json @@ -2630,28 +2630,6 @@ "reference": { "ignore_above": 1024, "type": "keyword" - }, - "subtechnique": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "fields": { - "text": { - "norms": false, - "type": "text" - } - }, - "ignore_above": 1024, - "type": "keyword" - }, - "reference": { - "ignore_above": 1024, - "type": "keyword" - } - } } } }