[Security Solution][Detections]Indicator Match Enrichment

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/ecs_mapping.json

@@ -2457,6 +2457,144 @@
             "ignore_above": 1024,
             "type": "keyword"
           },
+          "indicator": {
+            "type": "nested",
+            "properties": {
+              "as": {
+                "properties": {
+                  "number": {
+                    "type": "long"
+                  },
+                  "organization": {
+                    "properties": {
+                      "name": {
+                        "fields": {
+                          "text": {
+                            "norms": false,
+                            "type": "text"
+                          }
+                        },
+                        "ignore_above": 1024,
+                        "type": "keyword"
+                      }
+                    }
+                  }
+                }
+              },
+              "confidence": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "dataset": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "description": {
+                "type": "wildcard"
+              },
+              "domain": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "email": {
+                "properties": {
+                  "address": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "first_seen": {
+                "type": "date"
+              },
+              "geo": {
+                "properties": {
+                  "city_name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "continent_name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "country_iso_code": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "country_name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "location": {
+                    "type": "geo_point"
+                  },
+                  "name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "region_iso_code": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "region_name": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "ip": {
+                "type": "ip"
+              },
+              "last_seen": {
+                "type": "date"
+              },
+              "marking": {
+                "properties": {
+                  "tlp": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "matched": {
+                "properties": {
+                  "atomic": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "field": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "type": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
+              },
+              "module": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "port": {
+                "type": "long"
+              },
+              "provider": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              },
+              "scanner_stats": {
+                "type": "long"
+              },
+              "sightings": {
+                "type": "long"
+              },
+              "type": {
+                "ignore_above": 1024,
+                "type": "keyword"
+              }
+            }
+          },
           "tactic": {
             "properties": {
               "id": {
@@ -2492,6 +2630,28 @@
               "reference": {
                 "ignore_above": 1024,
                 "type": "keyword"
+              },
+              "subtechnique": {
+                "properties": {
+                  "id": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "name": {
+                    "fields": {
+                      "text": {
+                        "norms": false,
+                        "type": "text"
+                      }
+                    },
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  },
+                  "reference": {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
               }
             }
           }

x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts

@@ -8,7 +8,7 @@
 import signalsMapping from './signals_mapping.json';
 import ecsMapping from './ecs_mapping.json';
 
-export const SIGNALS_TEMPLATE_VERSION = 14;
+export const SIGNALS_TEMPLATE_VERSION = 15;
 export const MIN_EQL_RULE_INDEX_VERSION = 2;
 
 export const getSignalsTemplate = (index: string) => {

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts

@@ -14,6 +14,7 @@ import {
   repeatedSearchResultsWithSortId,
   repeatedSearchResultsWithNoSortId,
   sampleDocSearchResultsNoSortIdNoHits,
+  sampleDocWithSortId,
 } from './__mocks__/es_results';
 import { searchAfterAndBulkCreate } from './search_after_bulk_create';
 import { buildRuleMessageFactory } from './rule_messages';
@@ -870,4 +871,93 @@ describe('searchAfterAndBulkCreate', () => {
     expect(createdSignalsCount).toEqual(4);
     expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
   });
+
+  it('invokes the enrichment callback with signal search results', async () => {
+    const sampleParams = sampleRuleAlertParams(30);
+    mockService.callCluster
+      .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(0, 3)))
+      .mockResolvedValueOnce({
+        took: 100,
+        errors: false,
+        items: [
+          {
+            create: {
+              status: 201,
+            },
+          },
+        ],
+      })
+      .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(3, 6)))
+      .mockResolvedValueOnce({
+        took: 100,
+        errors: false,
+        items: [
+          {
+            create: {
+              status: 201,
+            },
+          },
+        ],
+      })
+      .mockResolvedValueOnce(repeatedSearchResultsWithSortId(4, 1, someGuids.slice(6, 9)))
+      .mockResolvedValueOnce({
+        took: 100,
+        errors: false,
+        items: [
+          {
+            create: {
+              status: 201,
+            },
+          },
+        ],
+      })
+      .mockResolvedValueOnce(sampleDocSearchResultsNoSortIdNoHits());
+
+    const mockEnrichment = jest.fn((a) => a);
+    const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({
+      enrichment: mockEnrichment,
+      ruleParams: sampleParams,
+      gap: moment.duration(2, 'm'),
+      previousStartedAt: moment().subtract(10, 'm').toDate(),
+      listClient,
+      exceptionsList: [],
+      services: mockService,
+      logger: mockLogger,
+      eventsTelemetry: undefined,
+      id: sampleRuleGuid,
+      inputIndexPattern,
+      signalsIndex: DEFAULT_SIGNALS_INDEX,
+      name: 'rule-name',
+      actions: [],
+      createdAt: '2020-01-28T15:58:34.810Z',
+      updatedAt: '2020-01-28T15:59:14.004Z',
+      createdBy: 'elastic',
+      updatedBy: 'elastic',
+      interval: '5m',
+      enabled: true,
+      pageSize: 1,
+      filter: undefined,
+      refresh: false,
+      tags: ['some fake tag 1', 'some fake tag 2'],
+      throttle: 'no_actions',
+      buildRuleMessage,
+    });
+
+    expect(mockEnrichment).toHaveBeenCalledWith(
+      expect.objectContaining({
+        hits: expect.objectContaining({
+          hits: expect.arrayContaining([
+            expect.objectContaining({
+              ...sampleDocWithSortId(),
+              _id: expect.any(String),
+            }),
+          ]),
+        }),
+      })
+    );
+    expect(success).toEqual(true);
+    expect(mockService.callCluster).toHaveBeenCalledTimes(7);
+    expect(createdSignalsCount).toEqual(3);
+    expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000'));
+  });
 });

x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts

@@ -7,6 +7,7 @@
 
 /* eslint-disable complexity */
 
+import { identity } from 'lodash';
 import { singleSearchAfter } from './single_search_after';
 import { singleBulkCreate } from './single_bulk_create';
 import { filterEventsAgainstList } from './filters/filter_events_against_list';
@@ -49,6 +50,7 @@ export const searchAfterAndBulkCreate = async ({
   tags,
   throttle,
   buildRuleMessage,
+  enrichment = identity,
 }: SearchAfterAndBulkCreateParams): Promise<SearchAfterAndBulkCreateReturnType> => {
   let toReturn = createSearchAfterReturnType();
 
@@ -106,7 +108,7 @@ export const searchAfterAndBulkCreate = async ({
             services,
             logger,
             filter,
-            pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result.
+            pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
             timestampOverride: ruleParams.timestampOverride,
             excludeDocsWithTimestampOverride: true,
           });
@@ -117,13 +119,16 @@ export const searchAfterAndBulkCreate = async ({
             backupSortId = lastSortId[0];
             hasBackupSortId = true;
           } else {
+            // TODO: This comment does not seem to match the code; this is the first loop and the first search, so how could the initial search result be known?
+            // answer: these were previously not blocking searches, and so the "second" search actually executed first TLDR outdated comment
             // if no sort id on backup search and the initial search result was also empty
             logger.debug(buildRuleMessage('backupSortIds was empty on searchResultB'));
             hasBackupSortId = false;
           }
 
           mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResultB]);
 
+          // TODO again, on the first loop this is the FIRST search and toReturn is default values
           // merge the search result from the secondary search with the first
           toReturn = mergeReturns([
             toReturn,
@@ -139,7 +144,6 @@ export const searchAfterAndBulkCreate = async ({
         }
 
         if (hasSortId) {
-          // only execute search if we have something to sort on or if it is the first search
           const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({
             buildRuleMessage,
             searchAfterSortId: sortId,
@@ -149,7 +153,7 @@ export const searchAfterAndBulkCreate = async ({
             services,
             logger,
             filter,
-            pageSize: tuple.maxSignals < pageSize ? Math.ceil(tuple.maxSignals) : pageSize, // maximum number of docs to receive per search result.
+            pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)),
             timestampOverride: ruleParams.timestampOverride,
             excludeDocsWithTimestampOverride: false,
           });
@@ -166,6 +170,7 @@ export const searchAfterAndBulkCreate = async ({
             }),
           ]);
 
+          // TODO it's unclear why hits are guaranteed here; the type appears to be identical to the previous result which has more guards
           // we are guaranteed to have searchResult hits at this point
           // because we check before if the totalHits or
           // searchResult.hits.hits.length is 0
@@ -186,14 +191,6 @@ export const searchAfterAndBulkCreate = async ({
           buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`)
         );
 
-        // search results yielded zero hits so exit
-        // with search_after, these two values can be different when
-        // searching with the last sortId of a consecutive search_after
-        // yields zero hits, but there were hits using the previous
-        // sortIds.
-        // e.g. totalHits was 156, index 50 of 100 results, do another search-after
-        // this time with a new sortId, index 22 of the remaining 56, get another sortId
-        // search with that sortId, total is still 156 but the hits.hits array is empty.
         if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) {
           logger.debug(
             buildRuleMessage(
@@ -228,6 +225,8 @@ export const searchAfterAndBulkCreate = async ({
               tuple.maxSignals - signalsCreatedCount
             );
           }
+          const enrichedEvents = await enrichment(filteredEvents);
+
           const {
             bulkCreateDuration: bulkDuration,
             createdItemsCount: createdCount,
@@ -236,7 +235,7 @@ export const searchAfterAndBulkCreate = async ({
             errors: bulkErrors,
           } = await singleBulkCreate({
             buildRuleMessage,
-            filteredEvents,
+            filteredEvents: enrichedEvents,
             ruleParams,
             services,
             logger,