From 88d2e1e5281734362cc5b3d9d58a1dc7454252a8 Mon Sep 17 00:00:00 2001 From: FrankHassanabad Date: Fri, 4 Jun 2021 12:21:52 -0600 Subject: [PATCH] Added more tests and fixed more types and removed more ts-expect-error --- .../signals/__mocks__/es_results.ts | 4 +- .../detection_engine/signals/utils.test.ts | 86 +++++++++++++++++-- .../security_and_spaces/tests/timestamps.ts | 19 +++- .../timestamp_override/mappings.json | 1 + .../timestamp_override_1/mappings.json | 1 + .../timestamp_override_2/mappings.json | 1 + .../timestamp_override_3/mappings.json | 1 + .../timestamp_override_4/data.json | 2 +- .../timestamp_override_4/mappings.json | 1 + .../timestamp_override_5/data.json | 14 +++ .../timestamp_override_5/mappings.json | 39 +++++++++ 11 files changed, 159 insertions(+), 10 deletions(-) create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json create mode 100644 x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts index 33a6878fcfb9c8..0fed141ca4dbcb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts @@ -562,7 +562,9 @@ export const sampleBulkCreateErrorResult = { export const sampleDocSearchResultsNoSortId = ( someUuid: string = sampleIdGuid -): SignalSearchResponse => ({ +): SignalSearchResponse & { + hits: { hits: Array> }; +} => ({ took: 10, timed_out: false, _shards: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index f49492939eeb10..60bf0ec337f3db 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -40,6 +40,7 @@ import { lastValidDate, calculateThresholdSignalUuid, buildChunkedOrFilter, + getValidDateFromDoc, } from './utils'; import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types'; import { @@ -54,6 +55,7 @@ import { sampleDocSearchResultsNoSortIdNoHits, repeatedSearchResultsWithSortId, sampleDocSearchResultsNoSortId, + sampleDocNoSortId, } from './__mocks__/es_results'; import { ShardError } from '../../types'; @@ -1172,7 +1174,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1186,7 +1187,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1200,7 +1200,6 @@ describe('utils', () => { test('It will not set an invalid date time stamp from an invalid @timestamp string', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid']; @@ -1216,7 +1215,6 @@ describe('utils', () => { describe('lastValidDate', () => { test('It returns undefined if the search result contains a null timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null; @@ -1227,7 +1225,6 @@ describe('utils', () => { test('It returns undefined if the search result contains a undefined timestamp', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined; @@ -1238,7 +1235,6 @@ describe('utils', () => { test('It returns undefined if the search result contains an invalid string value', () => { const searchResult = sampleDocSearchResultsNoSortId(); - // @ts-expect-error @elastic/elasticsearch _source is optional (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value'; if (searchResult.hits.hits[0].fields != null) { (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value']; @@ -1294,6 +1290,84 @@ describe('utils', () => { }); }); + describe('getValidDateFromDoc', () => { + test('It returns undefined if the search result contains a null timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = null; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = null; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains a undefined timestamp', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = undefined; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = undefined; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns undefined if the search result contains an invalid string value', () => { + const doc = sampleDocNoSortId(); + (doc._source['@timestamp'] as unknown) = 'invalid value'; + if (doc.fields != null) { + (doc.fields['@timestamp'] as unknown) = ['invalid value']; + } + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date).toEqual(undefined); + }); + + test('It returns normal date time if set', () => { + const doc = sampleDocNoSortId(); + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z'); + }); + + test('It returns date time from field if set there', () => { + const timestamp = '2020-10-07T19:27:19.136Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + '@timestamp': [timestamp], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: undefined }); + expect(date?.toISOString()).toEqual(timestamp); + }); + + test('It returns timestampOverride date time if set', () => { + const override = '2020-10-07T19:20:28.049Z'; + const doc = sampleDocNoSortId(); + doc._source.different_timestamp = new Date(override).toISOString(); + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + + test('It returns timestampOverride date time from fields if set on it', () => { + const override = '2020-10-07T19:36:31.110Z'; + let doc = sampleDocNoSortId(); + if (doc == null) { + throw new TypeError('Test requires one element'); + } + doc = { + ...doc, + fields: { + different_timestamp: [override], + }, + }; + const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' }); + expect(date?.toISOString()).toEqual(override); + }); + }); + describe('createSearchAfterReturnType', () => { test('createSearchAfterReturnType will return full object when nothing is passed', () => { const searchAfterReturnType = createSearchAfterReturnType(); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts index 8f93d305371f26..16610e6a449156 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts @@ -32,19 +32,21 @@ export default ({ getService }: FtrProviderContext) => { * partial errors happen correctly */ describe('timestamp tests', () => { - describe('Signals generated from events with a timestamp in milliseconds', () => { + describe('Signals generated from events with a timestamp in seconds is converted correctly into the forced ISO8601 format when copying', () => { beforeEach(async () => { await createSignalsIndex(supertest); await esArchiver.load('security_solution/timestamp_in_seconds'); + await esArchiver.load('security_solution/timestamp_override_5'); }); afterEach(async () => { await deleteSignalsIndex(supertest); await deleteAllAlerts(supertest); await esArchiver.unload('security_solution/timestamp_in_seconds'); + await esArchiver.unload('security_solution/timestamp_override_5'); }); - it('should convert the timestamp which is epoch_seconds into the correct ISO format', async () => { + it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => { const rule = getRuleForSignalTesting(['timestamp_in_seconds']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); @@ -53,6 +55,19 @@ export default ({ getService }: FtrProviderContext) => { const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); expect(hits).to.eql(['2021-06-02T23:33:15.000Z']); }); + + it('should still use the @timestamp field even with an override field. It should never use the override field', async () => { + const rule: QueryCreateSchema = { + ...getRuleForSignalTesting(['myfakeindex-5']), + timestamp_override: 'event.ingested', + }; + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 1, [id]); + const signalsOpen = await getSignalsByIds(supertest, [id]); + const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort(); + expect(hits).to.eql(['2020-12-16T15:16:18.000Z']); + }); }); /** diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json index 085ab34a3d58a7..092519a792863b 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json index 085ab34a3d58a7..092519a792863b 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-1", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json index 49a27a423cdaac..1f1c1673fe1a25 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-2", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json index 736584386a7052..a0409280c34ebc 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-3", "mappings": { + "dynamic": "strict", "properties": { "message": { "type": "text", diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json index ca7025b36154c8..ad0e7cbab7d2b0 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json @@ -6,7 +6,7 @@ "message": "hello world 4", "@timestamp": "2020-12-16T15:16:18.570Z", "event": { - "ingested": "2020-12-16T15:16:18.570Z" + "ingested": "2020-12-16T16:16:18.570Z" } }, "type": "_doc" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json index ab4edc9f300e19..a4e021e45ff9e6 100644 --- a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json @@ -3,6 +3,7 @@ "value": { "index": "myfakeindex-4", "mappings": { + "dynamic": "strict", "properties": { "@timestamp": { "type": "date" diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json new file mode 100644 index 00000000000000..f2c81e9b5e45e3 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json @@ -0,0 +1,14 @@ +{ + "type": "doc", + "value": { + "index": "myfakeindex-5", + "source": { + "@timestamp": 1608131778, + "message": "hello world 4", + "event": { + "ingested": 1622676795 + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json new file mode 100644 index 00000000000000..a9735aaeca8ef9 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json @@ -0,0 +1,39 @@ +{ + "type": "index", + "value": { + "index": "myfakeindex-5", + "mappings": { + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date", + "format": "epoch_second" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "event": { + "properties": { + "ingested": { + "type": "date", + "format": "epoch_second" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}