Added more tests and fixed more types and removed more ts-expect-error

elastic · Jun 4, 2021 · 88d2e1e · 88d2e1e
1 parent 02184dc
commit 88d2e1e
Show file tree

Hide file tree

Showing 11 changed files with 159 additions and 10 deletions.
diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
@@ -562,7 +562,9 @@ export const sampleBulkCreateErrorResult = {
 
 export const sampleDocSearchResultsNoSortId = (
   someUuid: string = sampleIdGuid
-): SignalSearchResponse => ({
+): SignalSearchResponse & {
+  hits: { hits: Array<ReturnType<typeof sampleDocNoSortId>> };
+} => ({
   took: 10,
   timed_out: false,
   _shards: {

diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts
@@ -40,6 +40,7 @@ import {
   lastValidDate,
   calculateThresholdSignalUuid,
   buildChunkedOrFilter,
+  getValidDateFromDoc,
 } from './utils';
 import { BulkResponseErrorAggregation, SearchAfterAndBulkCreateReturnType } from './types';
 import {
@@ -54,6 +55,7 @@ import {
   sampleDocSearchResultsNoSortIdNoHits,
   repeatedSearchResultsWithSortId,
   sampleDocSearchResultsNoSortId,
+  sampleDocNoSortId,
 } from './__mocks__/es_results';
 import { ShardError } from '../../types';
 
@@ -1172,7 +1174,6 @@ describe('utils', () => {
 
     test('It will not set an invalid date time stamp from a non-existent @timestamp when the index is not 100% ECS compliant', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined;
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined;
@@ -1186,7 +1187,6 @@ describe('utils', () => {
 
     test('It will not set an invalid date time stamp from a null @timestamp when the index is not 100% ECS compliant', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null;
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null;
@@ -1200,7 +1200,6 @@ describe('utils', () => {
 
     test('It will not set an invalid date time stamp from an invalid @timestamp string', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid';
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid'];
@@ -1216,7 +1215,6 @@ describe('utils', () => {
   describe('lastValidDate', () => {
     test('It returns undefined if the search result contains a null timestamp', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = null;
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = null;
@@ -1227,7 +1225,6 @@ describe('utils', () => {
 
     test('It returns undefined if the search result contains a undefined timestamp', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = undefined;
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = undefined;
@@ -1238,7 +1235,6 @@ describe('utils', () => {
 
     test('It returns undefined if the search result contains an invalid string value', () => {
       const searchResult = sampleDocSearchResultsNoSortId();
-      // @ts-expect-error @elastic/elasticsearch _source is optional
       (searchResult.hits.hits[0]._source['@timestamp'] as unknown) = 'invalid value';
       if (searchResult.hits.hits[0].fields != null) {
         (searchResult.hits.hits[0].fields['@timestamp'] as unknown) = ['invalid value'];
@@ -1294,6 +1290,84 @@ describe('utils', () => {
     });
   });
 
+  describe('getValidDateFromDoc', () => {
+    test('It returns undefined if the search result contains a null timestamp', () => {
+      const doc = sampleDocNoSortId();
+      (doc._source['@timestamp'] as unknown) = null;
+      if (doc.fields != null) {
+        (doc.fields['@timestamp'] as unknown) = null;
+      }
+      const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
+      expect(date).toEqual(undefined);
+    });
+
+    test('It returns undefined if the search result contains a undefined timestamp', () => {
+      const doc = sampleDocNoSortId();
+      (doc._source['@timestamp'] as unknown) = undefined;
+      if (doc.fields != null) {
+        (doc.fields['@timestamp'] as unknown) = undefined;
+      }
+      const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
+      expect(date).toEqual(undefined);
+    });
+
+    test('It returns undefined if the search result contains an invalid string value', () => {
+      const doc = sampleDocNoSortId();
+      (doc._source['@timestamp'] as unknown) = 'invalid value';
+      if (doc.fields != null) {
+        (doc.fields['@timestamp'] as unknown) = ['invalid value'];
+      }
+      const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
+      expect(date).toEqual(undefined);
+    });
+
+    test('It returns normal date time if set', () => {
+      const doc = sampleDocNoSortId();
+      const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
+      expect(date?.toISOString()).toEqual('2020-04-20T21:27:45.000Z');
+    });
+
+    test('It returns date time from field if set there', () => {
+      const timestamp = '2020-10-07T19:27:19.136Z';
+      let doc = sampleDocNoSortId();
+      if (doc == null) {
+        throw new TypeError('Test requires one element');
+      }
+      doc = {
+        ...doc,
+        fields: {
+          '@timestamp': [timestamp],
+        },
+      };
+      const date = getValidDateFromDoc({ doc, timestampOverride: undefined });
+      expect(date?.toISOString()).toEqual(timestamp);
+    });
+
+    test('It returns timestampOverride date time if set', () => {
+      const override = '2020-10-07T19:20:28.049Z';
+      const doc = sampleDocNoSortId();
+      doc._source.different_timestamp = new Date(override).toISOString();
+      const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' });
+      expect(date?.toISOString()).toEqual(override);
+    });
+
+    test('It returns timestampOverride date time from fields if set on it', () => {
+      const override = '2020-10-07T19:36:31.110Z';
+      let doc = sampleDocNoSortId();
+      if (doc == null) {
+        throw new TypeError('Test requires one element');
+      }
+      doc = {
+        ...doc,
+        fields: {
+          different_timestamp: [override],
+        },
+      };
+      const date = getValidDateFromDoc({ doc, timestampOverride: 'different_timestamp' });
+      expect(date?.toISOString()).toEqual(override);
+    });
+  });
+
   describe('createSearchAfterReturnType', () => {
     test('createSearchAfterReturnType will return full object when nothing is passed', () => {
       const searchAfterReturnType = createSearchAfterReturnType();

diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/timestamps.ts
@@ -32,19 +32,21 @@ export default ({ getService }: FtrProviderContext) => {
    * partial errors happen correctly
    */
   describe('timestamp tests', () => {
-    describe('Signals generated from events with a timestamp in milliseconds', () => {
+    describe('Signals generated from events with a timestamp in seconds is converted correctly into the forced ISO8601 format when copying', () => {
       beforeEach(async () => {
         await createSignalsIndex(supertest);
         await esArchiver.load('security_solution/timestamp_in_seconds');
+        await esArchiver.load('security_solution/timestamp_override_5');
       });
 
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
         await deleteAllAlerts(supertest);
         await esArchiver.unload('security_solution/timestamp_in_seconds');
+        await esArchiver.unload('security_solution/timestamp_override_5');
       });
 
-      it('should convert the timestamp which is epoch_seconds into the correct ISO format', async () => {
+      it('should convert the @timestamp which is epoch_seconds into the correct ISO format', async () => {
         const rule = getRuleForSignalTesting(['timestamp_in_seconds']);
         const { id } = await createRule(supertest, rule);
         await waitForRuleSuccessOrStatus(supertest, id);
@@ -53,6 +55,19 @@ export default ({ getService }: FtrProviderContext) => {
         const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
         expect(hits).to.eql(['2021-06-02T23:33:15.000Z']);
       });
+
+      it('should still use the @timestamp field even with an override field. It should never use the override field', async () => {
+        const rule: QueryCreateSchema = {
+          ...getRuleForSignalTesting(['myfakeindex-5']),
+          timestamp_override: 'event.ingested',
+        };
+        const { id } = await createRule(supertest, rule);
+        await waitForRuleSuccessOrStatus(supertest, id);
+        await waitForSignalsToBePresent(supertest, 1, [id]);
+        const signalsOpen = await getSignalsByIds(supertest, [id]);
+        const hits = signalsOpen.hits.hits.map((hit) => hit._source.signal.original_time).sort();
+        expect(hits).to.eql(['2020-12-16T15:16:18.000Z']);
+      });
     });
 
     /**

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override/mappings.json
@@ -3,6 +3,7 @@
   "value": {
     "index": "myfakeindex-1",
     "mappings": {
+      "dynamic": "strict",
       "properties": {
         "message": {
           "type": "text",

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_1/mappings.json
@@ -3,6 +3,7 @@
   "value": {
     "index": "myfakeindex-1",
     "mappings": {
+      "dynamic": "strict",
       "properties": {
         "message": {
           "type": "text",

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_2/mappings.json
@@ -3,6 +3,7 @@
   "value": {
     "index": "myfakeindex-2",
     "mappings": {
+      "dynamic": "strict",
       "properties": {
         "message": {
           "type": "text",

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_3/mappings.json
@@ -3,6 +3,7 @@
   "value": {
     "index": "myfakeindex-3",
     "mappings": {
+      "dynamic": "strict",
       "properties": {
         "message": {
           "type": "text",

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/data.json
@@ -6,7 +6,7 @@
       "message": "hello world 4",
       "@timestamp": "2020-12-16T15:16:18.570Z",
       "event": {
-        "ingested": "2020-12-16T15:16:18.570Z"
+        "ingested": "2020-12-16T16:16:18.570Z"
       }
     },
     "type": "_doc"

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_4/mappings.json
@@ -3,6 +3,7 @@
   "value": {
     "index": "myfakeindex-4",
     "mappings": {
+      "dynamic": "strict",
       "properties": {
         "@timestamp": {
           "type": "date"

diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/data.json
@@ -0,0 +1,14 @@
+{
+  "type": "doc",
+  "value": {
+    "index": "myfakeindex-5",
+    "source": {
+      "@timestamp": 1608131778,
+      "message": "hello world 4",
+      "event": {
+        "ingested": 1622676795
+      }
+    },
+    "type": "_doc"
+  }
+}
diff --git a/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json b/x-pack/test/functional/es_archives/security_solution/timestamp_override_5/mappings.json
@@ -0,0 +1,39 @@
+{
+  "type": "index",
+  "value": {
+    "index": "myfakeindex-5",
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "@timestamp": {
+          "type": "date",
+          "format": "epoch_second"
+        },
+        "message": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "event": {
+          "properties": {
+            "ingested": {
+              "type": "date",
+              "format": "epoch_second"
+            }
+          }
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "refresh_interval": "1s",
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}