From 00fc86fc7548c78b5a9b3e6d573bfe38f9023a22 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Wed, 15 Jul 2020 07:26:24 -0400 Subject: [PATCH] [Security Solution][Detection Engine] - Update exceptions logic (#71512) Co-authored-by: Elastic Machine Co-authored-by: Yara Tercero --- .../scripts/lists/new/items/ip_item.json | 2 +- .../scripts/lists/new/items/keyword_item.json | 2 +- .../build_exceptions_query.test.ts | 976 +++++------------- .../build_exceptions_query.ts | 118 +-- .../detection_engine/get_query_filter.test.ts | 130 +-- .../detection_engine/get_query_filter.ts | 16 +- .../common/detection_engine/utils.test.ts | 105 ++ .../common/detection_engine/utils.ts | 17 + .../signals/filter_events_with_list.ts | 20 +- .../signals/get_filter.test.ts | 85 +- .../signals/single_search_after.ts | 1 + .../detection_engine/signals/utils.test.ts | 49 - .../lib/detection_engine/signals/utils.ts | 8 +- 13 files changed, 562 insertions(+), 967 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/detection_engine/utils.test.ts create mode 100644 x-pack/plugins/security_solution/common/detection_engine/utils.ts diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json index 563139c40c0cad..c2238890496bbd 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/ip_item.json @@ -1,5 +1,5 @@ { "id": "ip_item", "list_id": "ip_list", - "value": "10.4.2.140" + "value": "127.0.0.1" } diff --git a/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json index 96d925c157490d..0848dc4c1bd94f 100644 --- a/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json +++ b/x-pack/plugins/lists/server/scripts/lists/new/items/keyword_item.json @@ -1,4 +1,4 @@ { "list_id": "keyword_list", - "value": "kibana" + "value": "zeek" } diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts index 26a219507c3aee..caf2dfb761ed0d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.test.ts @@ -113,226 +113,97 @@ describe('build_exceptions_query', () => { }); describe('operatorBuilder', () => { - describe("when 'exclude' is true", () => { - describe('and langauge is kuery', () => { - test('it returns "not " when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'kuery', exclude }); - expect(operator).toEqual('not '); - }); - test('it returns empty string when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'kuery', exclude }); - expect(operator).toEqual(''); - }); + describe('and language is kuery', () => { + test('it returns empty string when operator is "included"', () => { + const operator = operatorBuilder({ operator: 'included', language: 'kuery' }); + expect(operator).toEqual(''); }); - - describe('and language is lucene', () => { - test('it returns "NOT " when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'lucene', exclude }); - expect(operator).toEqual('NOT '); - }); - test('it returns empty string when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'lucene', exclude }); - expect(operator).toEqual(''); - }); + test('it returns "not " when operator is "excluded"', () => { + const operator = operatorBuilder({ operator: 'excluded', language: 'kuery' }); + expect(operator).toEqual('not '); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - describe('and language is kuery', () => { - test('it returns empty string when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'kuery', exclude }); - expect(operator).toEqual(''); - }); - test('it returns "not " when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'kuery', exclude }); - expect(operator).toEqual('not '); - }); + describe('and language is lucene', () => { + test('it returns empty string when operator is "included"', () => { + const operator = operatorBuilder({ operator: 'included', language: 'lucene' }); + expect(operator).toEqual(''); }); - - describe('and language is lucene', () => { - test('it returns empty string when operator is "included"', () => { - const operator = operatorBuilder({ operator: 'included', language: 'lucene', exclude }); - expect(operator).toEqual(''); - }); - test('it returns "NOT " when operator is "excluded"', () => { - const operator = operatorBuilder({ operator: 'excluded', language: 'lucene', exclude }); - expect(operator).toEqual('NOT '); - }); + test('it returns "NOT " when operator is "excluded"', () => { + const operator = operatorBuilder({ operator: 'excluded', language: 'lucene' }); + expect(operator).toEqual('NOT '); }); }); }); describe('buildExists', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:*'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:*'); + describe('kuery', () => { + test('it returns formatted wildcard string when operator is "excluded"', () => { + const query = buildExists({ + item: existsEntryWithExcluded, + language: 'kuery', }); + expect(query).toEqual('not host.name:*'); }); - - describe('lucene', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('_exists_host.name'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT _exists_host.name'); + test('it returns formatted wildcard string when operator is "included"', () => { + const query = buildExists({ + item: existsEntryWithIncluded, + language: 'kuery', }); + expect(query).toEqual('host.name:*'); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - - describe('kuery', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:*'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:*'); + describe('lucene', () => { + test('it returns formatted wildcard string when operator is "excluded"', () => { + const query = buildExists({ + item: existsEntryWithExcluded, + language: 'lucene', }); + expect(query).toEqual('NOT _exists_host.name'); }); - - describe('lucene', () => { - test('it returns formatted wildcard string when operator is "excluded"', () => { - const query = buildExists({ - item: existsEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT _exists_host.name'); - }); - test('it returns formatted wildcard string when operator is "included"', () => { - const query = buildExists({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('_exists_host.name'); + test('it returns formatted wildcard string when operator is "included"', () => { + const query = buildExists({ + item: existsEntryWithIncluded, + language: 'lucene', }); + expect(query).toEqual('_exists_host.name'); }); }); }); describe('buildMatch', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:suricata'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:suricata'); + describe('kuery', () => { + test('it returns formatted string when operator is "included"', () => { + const query = buildMatch({ + item: matchEntryWithIncluded, + language: 'kuery', }); + expect(query).toEqual('host.name:"suricata"'); }); - - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT host.name:suricata'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('host.name:suricata'); + test('it returns formatted string when operator is "excluded"', () => { + const query = buildMatch({ + item: matchEntryWithExcluded, + language: 'kuery', }); + expect(query).toEqual('not host.name:"suricata"'); }); }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; - }); - - describe('kuery', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('host.name:suricata'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'kuery', - exclude, - }); - expect(query).toEqual('not host.name:suricata'); + describe('lucene', () => { + test('it returns formatted string when operator is "included"', () => { + const query = buildMatch({ + item: matchEntryWithIncluded, + language: 'lucene', }); + expect(query).toEqual('host.name:"suricata"'); }); - - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const query = buildMatch({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('host.name:suricata'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const query = buildMatch({ - item: matchEntryWithExcluded, - language: 'lucene', - exclude, - }); - expect(query).toEqual('NOT host.name:suricata'); + test('it returns formatted string when operator is "excluded"', () => { + const query = buildMatch({ + item: matchEntryWithExcluded, + language: 'lucene', }); + expect(query).toEqual('NOT host.name:"suricata"'); }); }); }); @@ -352,152 +223,83 @@ describe('build_exceptions_query', () => { operator: 'excluded', }); - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns empty string if given an empty array for "values"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndNoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual(''); - }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:(suricata)'); - }); - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:(suricata or auditd)'); + describe('kuery', () => { + test('it returns empty string if given an empty array for "values"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndNoValues, + language: 'kuery', }); + expect(exceptionSegment).toEqual(''); + }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata or auditd)'); + test('it returns formatted string when "values" includes only one item', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndOneValue, + language: 'kuery', }); + + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata OR auditd)'); - }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata OR auditd)'); - }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata)'); + test('it returns formatted string when operator is "included"', () => { + const exceptionSegment = buildMatchAny({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'kuery', }); - }); - }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; + expect(exceptionSegment).toEqual('host.name:("suricata" or "auditd")'); }); - describe('kuery', () => { - test('it returns empty string if given an empty array for "values"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndNoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual(''); - }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata)'); - }); - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata or auditd)'); + test('it returns formatted string when operator is "excluded"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithExcludedAndTwoValues, + language: 'kuery', }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(exceptionSegment).toEqual('not host.name:(suricata or auditd)'); - }); + expect(exceptionSegment).toEqual('not host.name:("suricata" or "auditd")'); }); + }); - describe('lucene', () => { - test('it returns formatted string when operator is "included"', () => { - const exceptionSegment = buildMatchAny({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata OR auditd)'); + describe('lucene', () => { + test('it returns formatted string when operator is "included"', () => { + const exceptionSegment = buildMatchAny({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'lucene', }); - test('it returns formatted string when operator is "excluded"', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithExcludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('NOT host.name:(suricata OR auditd)'); + + expect(exceptionSegment).toEqual('host.name:("suricata" OR "auditd")'); + }); + test('it returns formatted string when operator is "excluded"', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithExcludedAndTwoValues, + language: 'lucene', }); - test('it returns formatted string when "values" includes only one item', () => { - const exceptionSegment = buildMatchAny({ - item: entryWithIncludedAndOneValue, - language: 'lucene', - exclude, - }); - expect(exceptionSegment).toEqual('host.name:(suricata)'); + + expect(exceptionSegment).toEqual('NOT host.name:("suricata" OR "auditd")'); + }); + test('it returns formatted string when "values" includes only one item', () => { + const exceptionSegment = buildMatchAny({ + item: entryWithIncludedAndOneValue, + language: 'lucene', }); + + expect(exceptionSegment).toEqual('host.name:("suricata")'); }); }); }); describe('buildNested', () => { + // NOTE: Only KQL supports nested describe('kuery', () => { test('it returns formatted query when one item in nested entry', () => { const item: EntryNested = { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'nestedField', operator: 'excluded' })], + entries: [makeMatchEntry({ field: 'nestedField', operator: 'included' })], }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-1 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" }'); }); test('it returns formatted query when multiple items in nested entry', () => { @@ -505,206 +307,128 @@ describe('build_exceptions_query', () => { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded' }), - makeMatchEntry({ field: 'nestedFieldB', operator: 'excluded', value: 'value-2' }), + makeMatchEntry({ field: 'nestedField', operator: 'included' }), + makeMatchEntry({ field: 'nestedFieldB', operator: 'included', value: 'value-2' }), ], }; const result = buildNested({ item, language: 'kuery' }); - expect(result).toEqual('parent:{ nestedField:value-1 and nestedFieldB:value-2 }'); - }); - }); - - // TODO: Does lucene support nested query syntax? - describe.skip('lucene', () => { - test('it returns formatted query when one item in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [makeMatchEntry({ field: 'nestedField', operator: 'excluded' })], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:value-1 }'); - }); - - test('it returns formatted query when multiple items in nested entry', () => { - const item: EntryNested = { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded' }), - makeMatchEntry({ field: 'nestedFieldB', operator: 'excluded', value: 'value-2' }), - ], - }; - const result = buildNested({ item, language: 'lucene' }); - - expect(result).toEqual('parent:{ nestedField:value-1 AND nestedFieldB:value-2 }'); + expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }'); }); }); }); describe('evaluateValues', () => { - describe("when 'exclude' is true", () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:*'); - }); - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:suricata'); - }); - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(result).toEqual('not host.name:(suricata or auditd)'); + describe('kuery', () => { + test('it returns formatted wildcard string when "type" is "exists"', () => { + const result = evaluateValues({ + item: existsEntryWithIncluded, + language: 'kuery', }); + expect(result).toEqual('host.name:*'); }); - describe('lucene', () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT _exists_host.name'); - }); - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT host.name:suricata'); - }); - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(result).toEqual('NOT host.name:(suricata OR auditd)'); - }); + test('it returns formatted string when "type" is "match"', () => { + const result = evaluateValues({ + item: matchEntryWithIncluded, + language: 'kuery', }); + expect(result).toEqual('host.name:"suricata"'); }); - }); - describe("when 'exclude' is false", () => { - beforeEach(() => { - exclude = false; + test('it returns formatted string when "type" is "match_any"', () => { + const result = evaluateValues({ + item: matchAnyEntryWithIncludedAndTwoValues, + language: 'kuery', + }); + expect(result).toEqual('host.name:("suricata" or "auditd")'); }); + }); + describe('lucene', () => { describe('kuery', () => { test('it returns formatted wildcard string when "type" is "exists"', () => { const result = evaluateValues({ item: existsEntryWithIncluded, - language: 'kuery', - exclude, + language: 'lucene', }); - expect(result).toEqual('host.name:*'); + expect(result).toEqual('_exists_host.name'); }); + test('it returns formatted string when "type" is "match"', () => { const result = evaluateValues({ item: matchEntryWithIncluded, - language: 'kuery', - exclude, + language: 'lucene', }); - expect(result).toEqual('host.name:suricata'); + expect(result).toEqual('host.name:"suricata"'); }); + test('it returns formatted string when "type" is "match_any"', () => { const result = evaluateValues({ item: matchAnyEntryWithIncludedAndTwoValues, - language: 'kuery', - exclude, - }); - expect(result).toEqual('host.name:(suricata or auditd)'); - }); - }); - - describe('lucene', () => { - describe('kuery', () => { - test('it returns formatted wildcard string when "type" is "exists"', () => { - const result = evaluateValues({ - item: existsEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('_exists_host.name'); - }); - test('it returns formatted string when "type" is "match"', () => { - const result = evaluateValues({ - item: matchEntryWithIncluded, - language: 'lucene', - exclude, - }); - expect(result).toEqual('host.name:suricata'); - }); - test('it returns formatted string when "type" is "match_any"', () => { - const result = evaluateValues({ - item: matchAnyEntryWithIncludedAndTwoValues, - language: 'lucene', - exclude, - }); - expect(result).toEqual('host.name:(suricata OR auditd)'); + language: 'lucene', }); + expect(result).toEqual('host.name:("suricata" OR "auditd")'); }); }); }); }); describe('formatQuery', () => { - describe('when query is empty string', () => { - test('it returns query if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], query: '', language: 'kuery' }); - expect(formattedQuery).toEqual(''); + describe('exclude is true', () => { + describe('when query is empty string', () => { + test('it returns empty string if "exceptions" is empty array', () => { + const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: true }); + expect(formattedQuery).toEqual(''); + }); + + test('it returns expected query string when single exception in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*'], + language: 'kuery', + exclude: true, + }); + expect(formattedQuery).toEqual('not ((b:("value-1" or "value-2") and not c:*))'); + }); }); - test('it returns expected query string when single exception in array', () => { + + test('it returns expected query string when multiple exceptions in array', () => { const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*'], - query: '', + exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], language: 'kuery', + exclude: true, }); - expect(formattedQuery).toEqual('(b:(value-1 or value-2) and not c:*)'); + expect(formattedQuery).toEqual( + 'not ((b:("value-1" or "value-2") and not c:*) or (not d:*))' + ); }); }); - test('it returns query if "exceptions" is empty array', () => { - const formattedQuery = formatQuery({ exceptions: [], query: 'a:*', language: 'kuery' }); - expect(formattedQuery).toEqual('a:*'); - }); + describe('exclude is false', () => { + describe('when query is empty string', () => { + test('it returns empty string if "exceptions" is empty array', () => { + const formattedQuery = formatQuery({ exceptions: [], language: 'kuery', exclude: false }); + expect(formattedQuery).toEqual(''); + }); - test('it returns expected query string when single exception in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*'], - query: 'a:*', - language: 'kuery', + test('it returns expected query string when single exception in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*'], + language: 'kuery', + exclude: false, + }); + expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*)'); + }); }); - expect(formattedQuery).toEqual('(a:* and b:(value-1 or value-2) and not c:*)'); - }); - test('it returns expected query string when multiple exceptions in array', () => { - const formattedQuery = formatQuery({ - exceptions: ['b:(value-1 or value-2) and not c:*', 'not d:*'], - query: 'a:*', - language: 'kuery', + test('it returns expected query string when multiple exceptions in array', () => { + const formattedQuery = formatQuery({ + exceptions: ['b:("value-1" or "value-2") and not c:*', 'not d:*'], + language: 'kuery', + exclude: false, + }); + expect(formattedQuery).toEqual('(b:("value-1" or "value-2") and not c:*) or (not d:*)'); }); - expect(formattedQuery).toEqual( - '(a:* and b:(value-1 or value-2) and not c:*) or (a:* and not d:*)' - ); }); }); @@ -712,81 +436,69 @@ describe('build_exceptions_query', () => { test('it returns empty string if empty lists array passed in', () => { const query = buildExceptionItemEntries({ language: 'kuery', - lists: [], - exclude, + entries: [], }); expect(query).toEqual(''); }); - test('it returns expected query when more than one item in list', () => { - // Equal to query && !(b && !c) -> (query AND NOT b) OR (query AND c) - // https://www.dcode.fr/boolean-expressions-calculator + test('it returns expected query when more than one item in exception item', () => { const payload: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists: payload, - exclude, + entries: payload, }); - const expectedQuery = 'not b:(value-1 or value-2) and c:value-3'; + const expectedQuery = 'b:("value-1" or "value-2") and not c:"value-3"'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list item includes nested value', () => { - // Equal to query && !(b || !c) -> (query AND NOT b AND c) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + test('it returns expected query when exception item includes nested value', () => { + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), + makeMatchEntry({ field: 'nestedField', operator: 'included', value: 'value-3' }), ], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:(value-1 or value-2) and parent:{ nestedField:value-3 }'; + const expectedQuery = 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" }'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list includes multiple items and nested "and" values', () => { - // Equal to query && !((b || !c) && d) -> (query AND NOT b AND c) OR (query AND NOT d) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + test('it returns expected query when exception item includes multiple items and nested "and" values', () => { + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), + makeMatchEntry({ field: 'nestedField', operator: 'included', value: 'value-3' }), ], }, makeExistsEntry({ field: 'd' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); const expectedQuery = - 'not b:(value-1 or value-2) and parent:{ nestedField:value-3 } and not d:*'; + 'b:("value-1" or "value-2") and parent:{ nestedField:"value-3" } and d:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when language is "lucene"', () => { - // Equal to query && !((b || !c) && !d) -> (query AND NOT b AND c) OR (query AND d) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), { field: 'parent', @@ -799,170 +511,56 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'lucene', - lists, - exclude, + entries, }); const expectedQuery = - 'NOT b:(value-1 OR value-2) AND parent:{ nestedField:value-3 } AND _exists_e'; + 'b:("value-1" OR "value-2") AND parent:{ nestedField:"value-3" } AND NOT _exists_e'; expect(query).toEqual(expectedQuery); }); - describe('when "exclude" is false', () => { - beforeEach(() => { - exclude = false; - }); - - test('it returns empty string if empty lists array passed in', () => { - const query = buildExceptionItemEntries({ - language: 'kuery', - lists: [], - exclude, - }); - - expect(query).toEqual(''); - }); - test('it returns expected query when more than one item in list', () => { - // Equal to query && !(b && !c) -> (query AND NOT b) OR (query AND c) - // https://www.dcode.fr/boolean-expressions-calculator - const payload: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-3' }), - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists: payload, - exclude, - }); - const expectedQuery = 'b:(value-1 or value-2) and not c:value-3'; - - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when list item includes nested value', () => { - // Equal to query && !(b || !c) -> (query AND NOT b AND c) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists, - exclude, - }); - const expectedQuery = 'b:(value-1 or value-2) and parent:{ nestedField:value-3 }'; - - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when list includes multiple items and nested "and" values', () => { - // Equal to query && !((b || !c) && d) -> (query AND NOT b AND c) OR (query AND NOT d) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - makeExistsEntry({ field: 'd' }), - ]; - const query = buildExceptionItemEntries({ - language: 'kuery', - lists, - exclude, - }); - const expectedQuery = 'b:(value-1 or value-2) and parent:{ nestedField:value-3 } and d:*'; - expect(query).toEqual(expectedQuery); - }); - - test('it returns expected query when language is "lucene"', () => { - // Equal to query && !((b || !c) && !d) -> (query AND NOT b AND c) OR (query AND d) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'nestedField', operator: 'excluded', value: 'value-3' }), - ], - }, - makeExistsEntry({ field: 'e', operator: 'excluded' }), - ]; - const query = buildExceptionItemEntries({ - language: 'lucene', - lists, - exclude, - }); - const expectedQuery = - 'b:(value-1 OR value-2) AND parent:{ nestedField:value-3 } AND NOT _exists_e'; - expect(query).toEqual(expectedQuery); - }); - }); - describe('exists', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [makeExistsEntry({ field: 'b' })]; + const entries: EntriesArray = [makeExistsEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:*'; + const expectedQuery = 'b:*'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; + const entries: EntriesArray = [makeExistsEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:*'; + const expectedQuery = 'not b:*'; expect(query).toEqual(expectedQuery); }); - test('it returns expected query when list includes list item with "and" values', () => { - // Equal to query && !(!b || !c) -> (query AND b AND c) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + test('it returns expected query when exception item includes entry item with "and" values', () => { + const entries: EntriesArray = [ makeExistsEntry({ field: 'b', operator: 'excluded' }), { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'c', operator: 'excluded', value: 'value-1' })], + entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'value-1' })], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:* and parent:{ c:value-1 }'; + const expectedQuery = 'not b:* and parent:{ c:"value-1" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> (query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeExistsEntry({ field: 'b' }), { field: 'parent', @@ -976,10 +574,9 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:* and parent:{ c:value-1 and d:value-2 } and not e:*'; + const expectedQuery = 'b:* and parent:{ c:"value-1" and d:"value-2" } and e:*'; expect(query).toEqual(expectedQuery); }); @@ -987,60 +584,49 @@ describe('build_exceptions_query', () => { describe('match', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; + const entries: EntriesArray = [makeMatchEntry({ field: 'b', value: 'value' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:value'; + const expectedQuery = 'b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', operator: 'excluded', value: 'value' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:value'; + const expectedQuery = 'not b:"value"'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with "and" values', () => { - // Equal to query && !(!b || !c) -> (query AND b AND c) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', operator: 'excluded', value: 'value' }), { field: 'parent', type: 'nested', - entries: [makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' })], + entries: [makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' })], }, ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:value and parent:{ c:valueC }'; + const expectedQuery = 'not b:"value" and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> (query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchEntry({ field: 'b', value: 'value' }), { field: 'parent', @@ -1054,10 +640,9 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:value and parent:{ c:valueC and d:valueD } and not e:valueE'; + const expectedQuery = 'b:"value" and parent:{ c:"valueC" and d:"valueD" } and e:"valueE"'; expect(query).toEqual(expectedQuery); }); @@ -1065,37 +650,29 @@ describe('build_exceptions_query', () => { describe('match_any', () => { test('it returns expected query when list includes single list item with operator of "included"', () => { - // Equal to query && !(b) -> (query AND NOT b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; + const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:(value-1 or value-2)'; + const expectedQuery = 'b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes single list item with operator of "excluded"', () => { - // Equal to query && !(!b) -> (query AND b) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; + const entries: EntriesArray = [makeMatchAnyEntry({ field: 'b', operator: 'excluded' })]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:(value-1 or value-2)'; + const expectedQuery = 'not b:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes list item with nested values', () => { - // Equal to query && !(!b || c) -> (query AND b AND NOT c) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b', operator: 'excluded' }), { field: 'parent', @@ -1105,27 +682,23 @@ describe('build_exceptions_query', () => { ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'b:(value-1 or value-2) and parent:{ c:valueC }'; + const expectedQuery = 'not b:("value-1" or "value-2") and parent:{ c:"valueC" }'; expect(query).toEqual(expectedQuery); }); test('it returns expected query when list includes multiple items', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator - const lists: EntriesArray = [ + const entries: EntriesArray = [ makeMatchAnyEntry({ field: 'b' }), makeMatchAnyEntry({ field: 'c' }), ]; const query = buildExceptionItemEntries({ language: 'kuery', - lists, - exclude, + entries, }); - const expectedQuery = 'not b:(value-1 or value-2) and not c:(value-1 or value-2)'; + const expectedQuery = 'b:("value-1" or "value-2") and c:("value-1" or "value-2")'; expect(query).toEqual(expectedQuery); }); @@ -1133,16 +706,19 @@ describe('build_exceptions_query', () => { }); describe('buildQueryExceptions', () => { - test('it returns original query if lists is empty array', () => { - const query = buildQueryExceptions({ query: 'host.name: *', language: 'kuery', lists: [] }); - const expectedQuery = 'host.name: *'; + test('it returns empty array if lists is empty array', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: [] }); - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); + expect(query).toEqual([]); + }); + + test('it returns empty array if lists is undefined', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: undefined }); + + expect(query).toEqual([]); }); test('it returns expected query when lists exist and language is "kuery"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1151,47 +727,33 @@ describe('build_exceptions_query', () => { field: 'parent', type: 'nested', entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), + makeMatchEntry({ field: 'c', operator: 'included', value: 'valueC' }), + makeMatchEntry({ field: 'd', operator: 'included', value: 'valueD' }), ], }, - makeMatchAnyEntry({ field: 'e' }), + makeMatchAnyEntry({ field: 'e', operator: 'excluded' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'kuery', lists: [payload, payload2], }); const expectedQuery = - '(a:* and some.parentField:{ nested.field:some value } and not some.not.nested.field:some value) or (a:* and not b:(value-1 or value-2) and parent:{ c:valueC and d:valueD } and not e:(value-1 or value-2))'; + 'not ((some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and not e:("value-1" or "value-2")))'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); + payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), - ], - }, - makeMatchAnyEntry({ field: 'e' }), - ]; + payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; const query = buildQueryExceptions({ - query: 'a:*', language: 'lucene', lists: [payload, payload2], }); const expectedQuery = - '(a:* AND some.parentField:{ nested.field:some value } AND NOT some.not.nested.field:some value) OR (a:* AND NOT b:(value-1 OR value-2) AND parent:{ c:valueC AND d:valueD } AND NOT e:(value-1 OR value-2))'; + 'NOT ((a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2")))'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); @@ -1201,21 +763,23 @@ describe('build_exceptions_query', () => { exclude = false; }); - test('it returns original query if lists is empty array', () => { + test('it returns empty array if lists is empty array', () => { const query = buildQueryExceptions({ - query: 'host.name: *', language: 'kuery', lists: [], exclude, }); - const expectedQuery = 'host.name: *'; - expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); + expect(query).toEqual([]); + }); + + test('it returns empty array if lists is undefined', () => { + const query = buildQueryExceptions({ language: 'kuery', lists: undefined, exclude }); + + expect(query).toEqual([]); }); test('it returns expected query when lists exist and language is "kuery"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); const payload2 = getExceptionListItemSchemaMock(); payload2.entries = [ @@ -1231,42 +795,28 @@ describe('build_exceptions_query', () => { makeMatchAnyEntry({ field: 'e' }), ]; const query = buildQueryExceptions({ - query: 'a:*', language: 'kuery', lists: [payload, payload2], exclude, }); const expectedQuery = - '(a:* and some.parentField:{ nested.field:some value } and some.not.nested.field:some value) or (a:* and b:(value-1 or value-2) and parent:{ c:valueC and d:valueD } and e:(value-1 or value-2))'; + '(some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value") or (b:("value-1" or "value-2") and parent:{ c:"valueC" and d:"valueD" } and e:("value-1" or "value-2"))'; expect(query).toEqual([{ query: expectedQuery, language: 'kuery' }]); }); test('it returns expected query when lists exist and language is "lucene"', () => { - // Equal to query && !((b || !c || d) && e) -> ((query AND NOT b AND c AND NOT d) OR (query AND NOT e) - // https://www.dcode.fr/boolean-expressions-calculator const payload = getExceptionListItemSchemaMock(); + payload.entries = [makeMatchAnyEntry({ field: 'a' }), makeMatchAnyEntry({ field: 'b' })]; const payload2 = getExceptionListItemSchemaMock(); - payload2.entries = [ - makeMatchAnyEntry({ field: 'b' }), - { - field: 'parent', - type: 'nested', - entries: [ - makeMatchEntry({ field: 'c', operator: 'excluded', value: 'valueC' }), - makeMatchEntry({ field: 'd', operator: 'excluded', value: 'valueD' }), - ], - }, - makeMatchAnyEntry({ field: 'e' }), - ]; + payload2.entries = [makeMatchAnyEntry({ field: 'c' }), makeMatchAnyEntry({ field: 'd' })]; const query = buildQueryExceptions({ - query: 'a:*', language: 'lucene', lists: [payload, payload2], exclude, }); const expectedQuery = - '(a:* AND some.parentField:{ nested.field:some value } AND some.not.nested.field:some value) OR (a:* AND b:(value-1 OR value-2) AND parent:{ c:valueC AND d:valueD } AND e:(value-1 OR value-2))'; + '(a:("value-1" OR "value-2") AND b:("value-1" OR "value-2")) OR (c:("value-1" OR "value-2") AND d:("value-1" OR "value-2"))'; expect(query).toEqual([{ query: expectedQuery, language: 'lucene' }]); }); diff --git a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts index a70e6a66385899..fc4fbae02b8fbd 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/build_exceptions_query.ts @@ -19,7 +19,8 @@ import { ExceptionListItemSchema, CreateExceptionListItemSchema, } from '../shared_imports'; -import { Language, Query } from './schemas/common/schemas'; +import { Language } from './schemas/common/schemas'; +import { hasLargeValueList } from './utils'; type Operators = 'and' | 'or' | 'not'; type LuceneOperators = 'AND' | 'OR' | 'NOT'; @@ -46,18 +47,16 @@ export const getLanguageBooleanOperator = ({ export const operatorBuilder = ({ operator, language, - exclude, }: { operator: Operator; language: Language; - exclude: boolean; }): string => { const not = getLanguageBooleanOperator({ language, value: 'not', }); - if ((exclude && operator === 'included') || (!exclude && operator === 'excluded')) { + if (operator === 'excluded') { return `${not} `; } else { return ''; @@ -67,14 +66,12 @@ export const operatorBuilder = ({ export const buildExists = ({ item, language, - exclude, }: { item: EntryExists; language: Language; - exclude: boolean; }): string => { const { operator, field } = item; - const exceptionOperator = operatorBuilder({ operator, language, exclude }); + const exceptionOperator = operatorBuilder({ operator, language }); switch (language) { case 'kuery': @@ -89,26 +86,22 @@ export const buildExists = ({ export const buildMatch = ({ item, language, - exclude, }: { item: EntryMatch; language: Language; - exclude: boolean; }): string => { const { value, operator, field } = item; - const exceptionOperator = operatorBuilder({ operator, language, exclude }); + const exceptionOperator = operatorBuilder({ operator, language }); - return `${exceptionOperator}${field}:${value}`; + return `${exceptionOperator}${field}:"${value}"`; }; export const buildMatchAny = ({ item, language, - exclude, }: { item: EntryMatchAny; language: Language; - exclude: boolean; }): string => { const { value, operator, field } = item; @@ -117,8 +110,8 @@ export const buildMatchAny = ({ return ''; default: const or = getLanguageBooleanOperator({ language, value: 'or' }); - const exceptionOperator = operatorBuilder({ operator, language, exclude }); - const matchAnyValues = value.map((v) => v); + const exceptionOperator = operatorBuilder({ operator, language }); + const matchAnyValues = value.map((v) => `"${v}"`); return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`; } @@ -133,7 +126,7 @@ export const buildNested = ({ }): string => { const { field, entries } = item; const and = getLanguageBooleanOperator({ language, value: 'and' }); - const values = entries.map((entry) => `${entry.field}:${entry.value}`); + const values = entries.map((entry) => `${entry.field}:"${entry.value}"`); return `${field}:{ ${values.join(` ${and} `)} }`; }; @@ -141,18 +134,16 @@ export const buildNested = ({ export const evaluateValues = ({ item, language, - exclude, }: { item: Entry | EntryNested; language: Language; - exclude: boolean; }): string => { if (entriesExists.is(item)) { - return buildExists({ item, language, exclude }); + return buildExists({ item, language }); } else if (entriesMatch.is(item)) { - return buildMatch({ item, language, exclude }); + return buildMatch({ item, language }); } else if (entriesMatchAny.is(item)) { - return buildMatchAny({ item, language, exclude }); + return buildMatchAny({ item, language }); } else if (entriesNested.is(item)) { return buildNested({ item, language }); } else { @@ -162,78 +153,79 @@ export const evaluateValues = ({ export const formatQuery = ({ exceptions, - query, language, + exclude, }: { exceptions: string[]; - query: string; language: Language; + exclude: boolean; }): string => { - if (exceptions.length > 0) { - const or = getLanguageBooleanOperator({ language, value: 'or' }); - const and = getLanguageBooleanOperator({ language, value: 'and' }); - const formattedExceptions = exceptions.map((exception) => { - if (query === '') { - return `(${exception})`; - } else { - return `(${query} ${and} ${exception})`; - } - }); - - return formattedExceptions.join(` ${or} `); - } else { - return query; + if (exceptions == null || (exceptions != null && exceptions.length === 0)) { + return ''; } + + const or = getLanguageBooleanOperator({ language, value: 'or' }); + const not = getLanguageBooleanOperator({ language, value: 'not' }); + const formattedExceptionItems = exceptions.map((exceptionItem, index) => { + if (index === 0) { + return `(${exceptionItem})`; + } + + return `${or} (${exceptionItem})`; + }); + + const exceptionItemsQuery = formattedExceptionItems.join(' '); + return exclude ? `${not} (${exceptionItemsQuery})` : exceptionItemsQuery; }; export const buildExceptionItemEntries = ({ - lists, + entries, language, - exclude, }: { - lists: EntriesArray; + entries: EntriesArray; language: Language; - exclude: boolean; }): string => { const and = getLanguageBooleanOperator({ language, value: 'and' }); - const exceptionItem = lists - .filter(({ type }) => type !== 'list') - .reduce((accum, listItem) => { - const exceptionSegment = evaluateValues({ item: listItem, language, exclude }); - return [...accum, exceptionSegment]; - }, []); - - return exceptionItem.join(` ${and} `); + const exceptionItemEntries = entries.reduce((accum, listItem) => { + const exceptionSegment = evaluateValues({ item: listItem, language }); + return [...accum, exceptionSegment]; + }, []); + + return exceptionItemEntries.join(` ${and} `); }; export const buildQueryExceptions = ({ - query, language, lists, exclude = true, }: { - query: Query; language: Language; lists: Array | undefined; exclude?: boolean; }): DataQuery[] => { - if (lists != null) { - const exceptions = lists.reduce((acc, exceptionItem) => { - return [ - ...acc, - ...(exceptionItem.entries !== undefined - ? [buildExceptionItemEntries({ lists: exceptionItem.entries, language, exclude })] - : []), - ]; - }, []); - const formattedQuery = formatQuery({ exceptions, language, query }); + if (lists == null || (lists != null && lists.length === 0)) { + return []; + } + + const exceptionItems = lists.reduce((acc, exceptionItem) => { + const { entries } = exceptionItem; + + if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) { + return [...acc, buildExceptionItemEntries({ entries, language })]; + } else { + return acc; + } + }, []); + + if (exceptionItems.length === 0) { + return []; + } else { + const formattedQuery = formatQuery({ exceptions: exceptionItems, language, exclude }); return [ { query: formattedQuery, language, }, ]; - } else { - return [{ query, language }]; } }; diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts index c19ef45605f83f..a8eb4e7bbb15bb 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts @@ -362,62 +362,45 @@ describe('get_filter', () => { expect(esQuery).toEqual({ bool: { filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { - filter: [ - { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'linux', - }, - }, - ], - }, - }, - { - bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, + must_not: { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', }, - ], - }, + }, + ], }, - score_mode: 'none', }, + score_mode: 'none', }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + }, + { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', }, }, - }, + ], }, - ], - }, + }, + ], }, - ], + }, }, }, ], @@ -469,52 +452,35 @@ describe('get_filter', () => { expect(esQuery).toEqual({ bool: { filter: [ + { bool: { minimum_should_match: 1, should: [{ match: { 'host.name': 'linux' } }] } }, { bool: { filter: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'linux', - }, + nested: { + path: 'some.parentField', + query: { + bool: { + minimum_should_match: 1, + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', + }, + }, + ], }, - ], + }, + score_mode: 'none', }, }, { bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, - }, - ], - }, - }, - score_mode: 'none', - }, - }, + minimum_should_match: 1, + should: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + match_phrase: { + 'some.not.nested.field': 'some value', }, }, ], diff --git a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts index 6584373b806d8e..a41589b5d0231d 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts @@ -31,12 +31,16 @@ export const getQueryFilter = ( title: index.join(), }; - const queries: DataQuery[] = buildQueryExceptions({ - query, - language, - lists, - exclude: excludeExceptions, - }); + const initialQuery = [{ query, language }]; + /* + * Pinning exceptions to 'kuery' because lucene + * does not support nested queries, while our exceptions + * UI does, since we can pass both lucene and kql into + * buildEsQuery, this allows us to offer nested queries + * regardless + */ + const exceptions = buildQueryExceptions({ language: 'kuery', lists, exclude: excludeExceptions }); + const queries: DataQuery[] = [...initialQuery, ...exceptions]; const config = { allowLeadingWildcards: true, diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts new file mode 100644 index 00000000000000..99680ffe41d444 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { hasLargeValueList, hasNestedEntry } from './utils'; +import { EntriesArray } from '../shared_imports'; + +describe('#hasLargeValueList', () => { + test('it returns false if empty array', () => { + const hasLists = hasLargeValueList([]); + + expect(hasLists).toBeFalsy(); + }); + + test('it returns true if item of type EntryList exists', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'list', + operator: 'included', + list: { id: 'some id', type: 'ip' }, + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasLargeValueList(entries); + + expect(hasLists).toBeTruthy(); + }); + + test('it returns false if item of type EntryList does not exist', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'match', + operator: 'included', + value: 'Elastic, N.V.', + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasLargeValueList(entries); + + expect(hasLists).toBeFalsy(); + }); +}); + +describe('#hasNestedEntry', () => { + test('it returns false if empty array', () => { + const hasLists = hasNestedEntry([]); + + expect(hasLists).toBeFalsy(); + }); + + test('it returns true if item of type EntryNested exists', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'nested', + entries: [ + { field: 'some field', type: 'match', operator: 'included', value: 'some value' }, + ], + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasNestedEntry(entries); + + expect(hasLists).toBeTruthy(); + }); + + test('it returns false if item of type EntryNested does not exist', () => { + const entries: EntriesArray = [ + { + field: 'actingProcess.file.signer', + type: 'match', + operator: 'included', + value: 'Elastic, N.V.', + }, + { + field: 'file.signature.signer', + type: 'match', + operator: 'excluded', + value: 'Global Signer', + }, + ]; + const hasLists = hasNestedEntry(entries); + + expect(hasLists).toBeFalsy(); + }); +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/utils.ts b/x-pack/plugins/security_solution/common/detection_engine/utils.ts new file mode 100644 index 00000000000000..fa1812235f897c --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/utils.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EntriesArray } from '../shared_imports'; + +export const hasLargeValueList = (entries: EntriesArray): boolean => { + const found = entries.filter(({ type }) => type === 'list'); + return found.length > 0; +}; + +export const hasNestedEntry = (entries: EntriesArray): boolean => { + const found = entries.filter(({ type }) => type === 'nested'); + return found.length > 0; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts index f16de8bf05ef41..ea52aecb379faf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts @@ -14,6 +14,7 @@ import { EntryList, ExceptionListItemSchema, } from '../../../../../lists/common/schemas'; +import { hasLargeValueList } from '../../../../common/detection_engine/utils'; interface FilterEventsAgainstList { listClient: ListClient; @@ -37,11 +38,28 @@ export const filterEventsAgainstList = async ({ return eventSearchResult; } + const exceptionItemsWithLargeValueLists = exceptionsList.reduce( + (acc, exception) => { + const { entries } = exception; + if (hasLargeValueList(entries)) { + return [...acc, exception]; + } + + return acc; + }, + [] + ); + + if (exceptionItemsWithLargeValueLists.length === 0) { + logger.debug(buildRuleMessage('about to return original search result')); + return eventSearchResult; + } + // narrow unioned type to be single const isStringableType = (val: SearchTypes) => ['string', 'number', 'boolean'].includes(typeof val); // grab the signals with values found in the given exception lists. - const filteredHitsPromises = exceptionsList.map( + const filteredHitsPromises = exceptionItemsWithLargeValueLists.map( async (exceptionItem: ExceptionListItemSchema) => { const { entries } = exceptionItem; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts index f34879781e0b02..a5740d7719f475 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/get_filter.test.ts @@ -192,71 +192,66 @@ describe('get_filter', () => { index: ['auditbeat-*'], lists: [getExceptionListItemSchemaMock()], }); + expect(filter).toEqual({ bool: { + must: [], filter: [ { bool: { - filter: [ + should: [ { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'host.name': 'siem', - }, - }, - ], + match: { + 'host.name': 'siem', }, }, - { - bool: { - filter: [ - { - nested: { - path: 'some.parentField', - query: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.parentField.nested.field': 'some value', - }, + ], + minimum_should_match: 1, + }, + }, + { + bool: { + must_not: { + bool: { + filter: [ + { + nested: { + path: 'some.parentField', + query: { + bool: { + should: [ + { + match_phrase: { + 'some.parentField.nested.field': 'some value', }, - ], - }, + }, + ], + minimum_should_match: 1, }, - score_mode: 'none', }, + score_mode: 'none', }, - { - bool: { - must_not: { - bool: { - minimum_should_match: 1, - should: [ - { - match: { - 'some.not.nested.field': 'some value', - }, - }, - ], + }, + { + bool: { + should: [ + { + match_phrase: { + 'some.not.nested.field': 'some value', }, }, - }, + ], + minimum_should_match: 1, }, - ], - }, + }, + ], }, - ], + }, }, }, ], - must: [], - must_not: [], should: [], + must_not: [], }, }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 5667f2e47b6d71..92ce7a28361155 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -52,6 +52,7 @@ export const singleSearchAfter = async ({ searchAfterSortId, timestampOverride, }); + const start = performance.now(); const nextSearchAfterResult: SignalSearchResponse = await services.callCluster( 'search', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index a6130a20f9c520..a610970907bf8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -9,7 +9,6 @@ import sinon from 'sinon'; import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks'; import { listMock } from '../../../../../lists/server/mocks'; -import { EntriesArray } from '../../../../common/shared_imports'; import { buildRuleMessageFactory } from './rule_messages'; import { ExceptionListClient } from '../../../../../lists/server'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; @@ -24,7 +23,6 @@ import { getGapMaxCatchupRatio, errorAggregator, getListsClient, - hasLargeValueList, getSignalTimeTuples, getExceptions, } from './utils'; @@ -585,53 +583,6 @@ describe('utils', () => { }); }); - describe('#hasLargeValueList', () => { - test('it returns false if empty array', () => { - const hasLists = hasLargeValueList([]); - - expect(hasLists).toBeFalsy(); - }); - - test('it returns true if item of type EntryList exists', () => { - const entries: EntriesArray = [ - { - field: 'actingProcess.file.signer', - type: 'list', - operator: 'included', - list: { id: 'some id', type: 'ip' }, - }, - { - field: 'file.signature.signer', - type: 'match', - operator: 'excluded', - value: 'Global Signer', - }, - ]; - const hasLists = hasLargeValueList(entries); - - expect(hasLists).toBeTruthy(); - }); - - test('it returns false if item of type EntryList does not exist', () => { - const entries: EntriesArray = [ - { - field: 'actingProcess.file.signer', - type: 'match', - operator: 'included', - value: 'Elastic, N.V.', - }, - { - field: 'file.signature.signer', - type: 'match', - operator: 'excluded', - value: 'Global Signer', - }, - ]; - const hasLists = hasLargeValueList(entries); - - expect(hasLists).toBeFalsy(); - }); - }); describe('getSignalTimeTuples', () => { test('should return a single tuple if no gap', () => { const someTuples = getSignalTimeTuples({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 0b95ff6786b013..1c59a4b7ea5d0f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -10,10 +10,11 @@ import dateMath from '@elastic/datemath'; import { Logger, SavedObjectsClientContract } from '../../../../../../../src/core/server'; import { AlertServices, parseDuration } from '../../../../../alerts/server'; import { ExceptionListClient, ListClient, ListPluginSetup } from '../../../../../lists/server'; -import { EntriesArray, ExceptionListItemSchema } from '../../../../../lists/common/schemas'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { ListArrayOrUndefined } from '../../../../common/detection_engine/schemas/types/lists'; import { BulkResponse, BulkResponseErrorAggregation, isValidUnit } from './types'; import { BuildRuleMessage } from './rule_messages'; +import { hasLargeValueList } from '../../../../common/detection_engine/utils'; interface SortExceptionsReturn { exceptionsWithValueLists: ExceptionListItemSchema[]; @@ -148,11 +149,6 @@ export const getListsClient = async ({ return { listClient, exceptionsClient }; }; -export const hasLargeValueList = (entries: EntriesArray): boolean => { - const found = entries.filter(({ type }) => type === 'list'); - return found.length > 0; -}; - export const getExceptions = async ({ client, lists,