From 7ac4f7d65c5330f7ac23946a08b28b173dd49fc3 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 3 Nov 2020 02:37:30 +0000 Subject: [PATCH 01/21] Fix threshold rule synthetic signal generation --- .../bulk_create_threshold_signals.test.ts | 119 ++++++++++++++++-- .../signals/bulk_create_threshold_signals.ts | 37 ++++-- .../signals/find_threshold_signals.ts | 2 +- 3 files changed, 139 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts index d97dc4ba2cbd24..6a75d0655cf59d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts @@ -4,10 +4,29 @@ * you may not use this file except in compliance with the Elastic License. */ +import { sampleDocNoSortIdNoVersion } from './__mocks__/es_results'; import { getThresholdSignalQueryFields } from './bulk_create_threshold_signals'; describe('getThresholdSignalQueryFields', () => { it('should return proper fields for match_phrase filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + event: { + dataset: 'traefik.access', + module: 'traefik', + }, + traefik: { + access: { + entryPointName: 'web-secure', + }, + }, + url: { + domain: 'kibana.siem.estc.dev', + }, + }, + }; const mockFilters = { bool: { must: [], @@ -71,15 +90,28 @@ describe('getThresholdSignalQueryFields', () => { }, }; - expect(getThresholdSignalQueryFields(mockFilters)).toEqual({ - 'event.module': 'traefik', + expect(getThresholdSignalQueryFields(mockHit, mockFilters)).toEqual({ 'event.dataset': 'traefik.access', + 'event.module': 'traefik', 'traefik.access.entryPointName': 'web-secure', 'url.domain': 'kibana.siem.estc.dev', }); }); it('should return proper fields object for nested match filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + event: { + dataset: 'traefik.access', + module: 'traefik', + }, + url: { + domain: 'kibana.siem.estc.dev', + }, + }, + }; const filters = { bool: { must: [], @@ -104,7 +136,7 @@ describe('getThresholdSignalQueryFields', () => { should: [ { match: { - 'event.dataset': 'traefik.access', + 'event.dataset': 'traefik.*', }, }, ], @@ -120,13 +152,23 @@ describe('getThresholdSignalQueryFields', () => { }, }; - expect(getThresholdSignalQueryFields(filters)).toEqual({ - 'event.module': 'traefik', + expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ 'event.dataset': 'traefik.access', + 'event.module': 'traefik', }); }); it('should return proper object for simple match filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + event: { + dataset: 'traefik.access', + module: 'traefik', + }, + }, + }; const filters = { bool: { must: [], @@ -154,13 +196,23 @@ describe('getThresholdSignalQueryFields', () => { }, }; - expect(getThresholdSignalQueryFields(filters)).toEqual({ - 'event.module': 'traefik', + expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ 'event.dataset': 'traefik.access', + 'event.module': 'traefik', }); }); it('should return proper object for simple match_phrase filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + event: { + dataset: 'traefik.access', + module: 'traefik', + }, + }, + }; const filters = { bool: { must: [], @@ -188,13 +240,22 @@ describe('getThresholdSignalQueryFields', () => { }, }; - expect(getThresholdSignalQueryFields(filters)).toEqual({ + expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ 'event.module': 'traefik', 'event.dataset': 'traefik.access', }); }); it('should return proper object for exists filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + event: { + module: 'traefik', + }, + }, + }; const filters = { bool: { should: [ @@ -226,6 +287,46 @@ describe('getThresholdSignalQueryFields', () => { minimum_should_match: 1, }, }; - expect(getThresholdSignalQueryFields(filters)).toEqual({}); + expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({}); + }); + + it('should NOT add invalid characters from CIDR such as the "/" proper object for simple match_phrase filters', () => { + const mockHit = { + ...sampleDocNoSortIdNoVersion(), + _source: { + '@timestamp': '2020-11-03T02:31:47.431Z', + destination: { + ip: '192.168.0.16', + }, + event: { + module: 'traefik', + }, + }, + }; + const filters = { + bool: { + must: [], + filter: [ + { + bool: { + should: [ + { + match: { + 'destination.ip': '192.168.0.0/16', + }, + }, + ], + minimum_should_match: 1, + }, + }, + ], + should: [], + must_not: [], + }, + }; + + expect(getThresholdSignalQueryFields(mockHit, filters)).toEqual({ + 'destination.ip': '192.168.0.16', + }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 9eee04030a909b..24c5c4e686bddb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -14,7 +14,7 @@ import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { SignalSearchResponse } from './types'; +import { SignalSearchResponse, SignalSourceHit } from './types'; import { BuildRuleMessage } from './rule_messages'; // used to generate constant Threshold Signals ID when run with the same params @@ -51,11 +51,25 @@ interface FilterObject { }; } -const getNestedQueryFilters = (filtersObj: FilterObject): Record => { +const injectFirstMatch = ( + hit: SignalSourceHit, + match: object | Record +): Record | undefined => { + if (match != null) { + for (const key of Object.keys(match)) { + return { [key]: get(key, hit._source) } as Record; + } + } +}; + +const getNestedQueryFilters = ( + hit: SignalSourceHit, + filtersObj: FilterObject +): Record => { if (Array.isArray(filtersObj.bool?.filter)) { return reduce( (acc, filterItem) => { - const nestedFilter = getNestedQueryFilters(filterItem); + const nestedFilter = getNestedQueryFilters(hit, filterItem); if (nestedFilter) { return { ...acc, ...nestedFilter }; @@ -70,27 +84,32 @@ const getNestedQueryFilters = (filtersObj: FilterObject): Record return ( (filtersObj.bool?.should && filtersObj.bool?.should[0] && - (filtersObj.bool.should[0].match || filtersObj.bool.should[0].match_phrase)) ?? + (injectFirstMatch(hit, filtersObj.bool.should[0].match) || + injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase))) ?? {} ); } }; -export const getThresholdSignalQueryFields = (filter: unknown) => { +export const getThresholdSignalQueryFields = (hit: SignalSourceHit, filter: unknown) => { const filters = get('bool.filter', filter); return reduce( (acc, item) => { if (item.match_phrase) { - return { ...acc, ...item.match_phrase }; + return { ...acc, ...injectFirstMatch(hit, item.match_phrase) }; } if (item.bool?.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { - return { ...acc, ...(item.bool.should[0].match || item.bool.should[0].match_phrase) }; + return { + ...acc, + ...(injectFirstMatch(hit, item.bool.should[0].match) || + injectFirstMatch(hit, item.bool.should[0].match_phrase)), + }; } if (item.bool?.filter) { - return { ...acc, ...getNestedQueryFilters(item) }; + return { ...acc, ...getNestedQueryFilters(hit, item) }; } return acc; @@ -163,7 +182,7 @@ export const transformThresholdResultsToEcs = ( threshold: Threshold, ruleId: string ): SignalSearchResponse => { - const signalQueryFields = getThresholdSignalQueryFields(filter); + const signalQueryFields = getThresholdSignalQueryFields(results.hits.hits[0], filter); const transformedHits = getTransformedHits( results, inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index b34825b92ae90c..aa045cf9023ddf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -66,7 +66,7 @@ export const findThresholdSignals = async ({ services, logger, filter, - pageSize: 0, + pageSize: 1, buildRuleMessage, }); }; From 0c9d5e3a0ade0eaad3ae8d0b2b275e9f64801783 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 5 Nov 2020 19:19:04 +0000 Subject: [PATCH 02/21] Use top_hits aggregation --- .../signals/bulk_create_threshold_signals.ts | 48 +++++++++-------- .../signals/find_threshold_signals.ts | 18 ++++++- .../lib/detection_engine/signals/types.ts | 6 ++- .../security_solution/server/lib/types.ts | 53 +++++++++++-------- 4 files changed, 79 insertions(+), 46 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 24c5c4e686bddb..2870b1ef5926a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -14,7 +14,7 @@ import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; -import { SignalSearchResponse, SignalSourceHit } from './types'; +import { SignalSearchResponse, SignalSourceHit, ThresholdAggregationBucket } from './types'; import { BuildRuleMessage } from './rule_messages'; // used to generate constant Threshold Signals ID when run with the same params @@ -125,7 +125,7 @@ const getTransformedHits = ( startedAt: Date, threshold: Threshold, ruleId: string, - signalQueryFields: Record + filter: unknown ) => { if (isEmpty(threshold.field)) { const totalResults = @@ -138,7 +138,8 @@ const getTransformedHits = ( const source = { '@timestamp': new Date().toISOString(), threshold_count: totalResults, - ...signalQueryFields, + // TODO: how to get signal query fields for this case??? + // ...signalQueryFields, }; return [ @@ -154,24 +155,30 @@ const getTransformedHits = ( return []; } - return results.aggregations.threshold.buckets.map( - // eslint-disable-next-line @typescript-eslint/naming-convention - ({ key, doc_count }: { key: string; doc_count: number }) => { - const source = { - '@timestamp': new Date().toISOString(), - threshold_count: doc_count, - ...signalQueryFields, - }; + return results.aggregations.threshold.buckets + .map( + ({ key, doc_count: docCount, top_threshold_hits: topHits }: ThresholdAggregationBucket) => { + const hit = topHits.hits.hits[0]; + if (hit == null) { + return null; + } - set(source, threshold.field, key); + const source = { + '@timestamp': new Date().toISOString(), // TODO: use timestamp of latest event? + threshold_count: docCount, + ...getThresholdSignalQueryFields(hit, filter), + }; - return { - _index: inputIndex, - _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), - _source: source, - }; - } - ); + set(source, threshold.field, key); + + return { + _index: inputIndex, + _id: uuidv5(`${ruleId}${startedAt}${threshold.field}${key}`, NAMESPACE_ID), + _source: source, + }; + } + ) + .filter((bucket: ThresholdAggregationBucket) => bucket != null); }; export const transformThresholdResultsToEcs = ( @@ -182,14 +189,13 @@ export const transformThresholdResultsToEcs = ( threshold: Threshold, ruleId: string ): SignalSearchResponse => { - const signalQueryFields = getThresholdSignalQueryFields(results.hits.hits[0], filter); const transformedHits = getTransformedHits( results, inputIndex, startedAt, threshold, ruleId, - signalQueryFields + filter ); const thresholdResults = { ...results, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index aa045cf9023ddf..034576ff31ebb5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -52,6 +52,22 @@ export const findThresholdSignals = async ({ field: threshold.field, min_doc_count: threshold.value, }, + aggs: { + // Get the most recent hit per bucket + top_threshold_hits: { + top_hits: { + sort: [ + { + '@timestamp': { + // TODO: custom timestamp fields??? + order: 'desc', + }, + }, + ], + size: 1, + }, + }, + }, }, } : {}; @@ -66,7 +82,7 @@ export const findThresholdSignals = async ({ services, logger, filter, - pageSize: 1, + pageSize: 0, buildRuleMessage, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9d4e7d8a810515..e9df7e79dcf0b6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -14,7 +14,7 @@ import { AlertExecutorOptions, AlertServices, } from '../../../../../alerts/server'; -import { SearchResponse } from '../../types'; +import { BaseSearchResponse, SearchResponse, TermAggregationBucket } from '../../types'; import { EqlSearchResponse, BaseHit, @@ -234,3 +234,7 @@ export interface SearchAfterAndBulkCreateReturnType { createdSignalsCount: number; errors: string[]; } + +export interface ThresholdAggregationBucket extends TermAggregationBucket { + top_threshold_hits: BaseSearchResponse; +} diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index 67967f2a3cc7e1..618710ebd5fc6d 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -69,41 +69,48 @@ export type ShardError = Partial<{ }>; }>; -export interface SearchResponse { +export interface SearchHits { + total: TotalValue | number; + max_score: number; + hits: Array< + BaseHit & { + _type: string; + _score: number; + _version?: number; + _explanation?: Explanation; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + highlight?: any; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inner_hits?: any; + matched_queries?: string[]; + sort?: string[]; + } + >; +} + +export interface BaseSearchResponse { + hits: SearchHits; +} + +export interface SearchResponse extends BaseSearchResponse { took: number; timed_out: boolean; _scroll_id?: string; _shards: ShardsResponse; - hits: { - total: TotalValue | number; - max_score: number; - hits: Array< - BaseHit & { - _type: string; - _score: number; - _version?: number; - _explanation?: Explanation; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - highlight?: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - inner_hits?: any; - matched_queries?: string[]; - sort?: string[]; - } - >; - }; // eslint-disable-next-line @typescript-eslint/no-explicit-any aggregations?: any; } export type SearchHit = SearchResponse['hits']['hits'][0]; +export interface TermAggregationBucket { + key: string; + doc_count: number; +} + export interface TermAggregation { [agg: string]: { - buckets: Array<{ - key: string; - doc_count: number; - }>; + buckets: TermAggregationBucket[]; }; } From 2f61b30d21d8912535695570ca196be7f3f2ce72 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Fri, 6 Nov 2020 01:57:12 +0000 Subject: [PATCH 03/21] Find signals and aggregate over search terms --- .../find_previous_threshold_signals.ts | 89 +++++++++++++++++++ .../signals/signal_rule_alert_type.ts | 44 ++++++++- 2 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts new file mode 100644 index 00000000000000..680ec1dd140dfc --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { singleSearchAfter } from './single_search_after'; + +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { SignalSearchResponse } from './types'; +import { BuildRuleMessage } from './rule_messages'; + +interface FindPreviousThresholdSignalsParams { + from: string; + to: string; + indexPattern: string[]; + services: AlertServices; + logger: Logger; + ruleId: string; + queryFields: string[]; + timestampOverride: TimestampOverrideOrUndefined; + buildRuleMessage: BuildRuleMessage; +} + +export const findPreviousThresholdSignals = async ({ + from, + to, + indexPattern, + services, + logger, + ruleId, + queryFields, + timestampOverride, + buildRuleMessage, +}: FindPreviousThresholdSignalsParams): Promise<{ + searchResult: SignalSearchResponse; + searchDuration: string; + searchErrors: string[]; +}> => { + const aggregations = { + rule: { + filter: { + term: { + 'signal.rule.id': ruleId, + }, + }, + aggs: { + threshold: { + terms: { + field: queryFields[0], // FIXME + }, + aggs: { + lastSignalTimestamp: { + max: { + field: '@timestamp', // TODO: or timestampOverride? Or signal.original_time? + }, + }, + previousSignalCount: { + sum: { + field: 'signal.threshold_count', + }, + }, + }, + }, + }, + }, + }; + + const filter = { + match_all: {}, + }; + + // TODO: paginate + return singleSearchAfter({ + aggregations, + searchAfterSortId: undefined, + timestampOverride, + index: indexPattern, + from, + to, + services, + logger, + filter, + pageSize: 0, + buildRuleMessage, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index a0d5c833b208cb..e885b1ef7aed8f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -46,8 +46,12 @@ import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; +import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; -import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; +import { + bulkCreateThresholdSignals, + getThresholdSignalQueryFields, +} from './bulk_create_threshold_signals'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -298,6 +302,42 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); + const fakeHit = { + _index: '', + _id: '', + _score: 1.0, + _source: { + '@timestamp': '', + }, + _type: '', + }; + const queryFields = getThresholdSignalQueryFields(fakeHit, esFilter); + + const { + searchResult: previousSignals, + searchErrors: previousSearchErrors, + } = await findPreviousThresholdSignals({ + indexPattern: [outputIndex], + from, + to, + services, + logger, + ruleId, + queryFields: Object.keys(queryFields), + timestampOverride, + buildRuleMessage, + }); + + // TODO: error checking + + previousSignals.aggregations.rule.aggregations.threshold.buckets.foreach( + (signal: object) => { + // TODO + } + ); + + // console.log(previousSignals); + const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, @@ -346,7 +386,7 @@ export const signalRulesAlertType = ({ }), createSearchAfterReturnType({ success, - errors: [...errors, ...searchErrors], + errors: [...errors, ...previousSearchErrors, ...searchErrors], createdSignalsCount: createdItemsCount, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), From 4610643896e332f38768fce8cde422576805a38e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 9 Nov 2020 20:11:00 +0000 Subject: [PATCH 04/21] Exclude dupes --- .../find_previous_threshold_signals.ts | 2 +- .../signals/signal_rule_alert_type.ts | 33 ++++++++++++++----- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 680ec1dd140dfc..ddbfbdd25e9e26 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -43,7 +43,7 @@ export const findPreviousThresholdSignals = async ({ rule: { filter: { term: { - 'signal.rule.id': ruleId, + 'signal.rule.rule_id': ruleId, }, }, aggs: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 5a9dbfa6be7cc3..b4daa13735aa7b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -330,15 +330,32 @@ export const signalRulesAlertType = ({ buildRuleMessage, }); - // TODO: error checking - - previousSignals.aggregations.rule.aggregations.threshold.buckets.foreach( - (signal: object) => { - // TODO - } - ); + previousSignals.aggregations.rule.threshold.buckets.forEach((bucket: object) => { + esFilter.bool.filter.push({ + bool: { + must_not: { + bool: { + must: [ + { + term: { + [Object.keys(queryFields)[0]]: bucket.key, + }, + }, + { + range: { + '@timestamp': { + lt: bucket.lastSignalTimestamp.value_as_string, + }, + }, + }, + ], + }, + }, + }, + }); + }); - // console.log(previousSignals); + // console.log(JSON.stringify(esFilter)); const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, From 3fa0e699a6bc538aac59efbdd3412437e03b8af3 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 01:44:29 +0000 Subject: [PATCH 05/21] Fixes to algorithm --- .../signals/bulk_create_threshold_signals.ts | 18 ++++++++++---- .../find_previous_threshold_signals.ts | 11 +++------ .../signals/signal_rule_alert_type.ts | 24 ++++--------------- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 2870b1ef5926a2..a05151cffe35eb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -53,9 +53,12 @@ interface FilterObject { const injectFirstMatch = ( hit: SignalSourceHit, - match: object | Record + match: string | object | Record ): Record | undefined => { if (match != null) { + if (typeof match === 'string') { + return { exists: get(match, hit._source) } as Record; + } for (const key of Object.keys(match)) { return { [key]: get(key, hit._source) } as Record; } @@ -85,7 +88,8 @@ const getNestedQueryFilters = ( (filtersObj.bool?.should && filtersObj.bool?.should[0] && (injectFirstMatch(hit, filtersObj.bool.should[0].match) || - injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase))) ?? + injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase) || + injectFirstMatch(hit, filtersObj.bool.should[0].exists.field))) ?? {} ); } @@ -100,11 +104,17 @@ export const getThresholdSignalQueryFields = (hit: SignalSourceHit, filter: unkn return { ...acc, ...injectFirstMatch(hit, item.match_phrase) }; } - if (item.bool?.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { + if ( + item.bool?.should && + (item.bool.should[0].match || + item.bool.should[0].match_phrase || + item.bool.should[0].exists) + ) { return { ...acc, ...(injectFirstMatch(hit, item.bool.should[0].match) || - injectFirstMatch(hit, item.bool.should[0].match_phrase)), + injectFirstMatch(hit, item.bool.should[0].match_phrase) || + injectFirstMatch(hit, item.bool.should[0].exists.field)), }; } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index ddbfbdd25e9e26..208524a425e4f3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -19,7 +19,7 @@ interface FindPreviousThresholdSignalsParams { services: AlertServices; logger: Logger; ruleId: string; - queryFields: string[]; + bucketByField: string; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; } @@ -31,7 +31,7 @@ export const findPreviousThresholdSignals = async ({ services, logger, ruleId, - queryFields, + bucketByField, timestampOverride, buildRuleMessage, }: FindPreviousThresholdSignalsParams): Promise<{ @@ -49,7 +49,7 @@ export const findPreviousThresholdSignals = async ({ aggs: { threshold: { terms: { - field: queryFields[0], // FIXME + field: bucketByField, }, aggs: { lastSignalTimestamp: { @@ -57,11 +57,6 @@ export const findPreviousThresholdSignals = async ({ field: '@timestamp', // TODO: or timestampOverride? Or signal.original_time? }, }, - previousSignalCount: { - sum: { - field: 'signal.threshold_count', - }, - }, }, }, }, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index b4daa13735aa7b..db292b88f25c5a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -48,10 +48,7 @@ import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; -import { - bulkCreateThresholdSignals, - getThresholdSignalQueryFields, -} from './bulk_create_threshold_signals'; +import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; import { scheduleNotificationActions, NotificationRuleTypeParams, @@ -304,17 +301,6 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); - const fakeHit = { - _index: '', - _id: '', - _score: 1.0, - _source: { - '@timestamp': '', - }, - _type: '', - }; - const queryFields = getThresholdSignalQueryFields(fakeHit, esFilter); - const { searchResult: previousSignals, searchErrors: previousSearchErrors, @@ -325,7 +311,7 @@ export const signalRulesAlertType = ({ services, logger, ruleId, - queryFields: Object.keys(queryFields), + bucketByField: threshold.field, timestampOverride, buildRuleMessage, }); @@ -338,13 +324,13 @@ export const signalRulesAlertType = ({ must: [ { term: { - [Object.keys(queryFields)[0]]: bucket.key, + [threshold.field]: bucket.key, }, }, { range: { '@timestamp': { - lt: bucket.lastSignalTimestamp.value_as_string, + lte: bucket.lastSignalTimestamp.value_as_string, }, }, }, @@ -355,8 +341,6 @@ export const signalRulesAlertType = ({ }); }); - // console.log(JSON.stringify(esFilter)); - const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, From 35e2119368f6cd48dccee0059d37e615e2ff677c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 15:17:24 +0000 Subject: [PATCH 06/21] Sync timestamps with events/signals --- .../detection_engine/signals/find_previous_threshold_signals.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 208524a425e4f3..41b108e096fc8c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -54,7 +54,7 @@ export const findPreviousThresholdSignals = async ({ aggs: { lastSignalTimestamp: { max: { - field: '@timestamp', // TODO: or timestampOverride? Or signal.original_time? + field: 'signal.original_time', // timestamp of last event captured by bucket }, }, }, From 5914e99ac80c305ceb77a2fd627ea9ce9eafc918 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 15:32:26 +0000 Subject: [PATCH 07/21] Add timestampOverride --- .../lib/detection_engine/signals/find_threshold_signals.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 034576ff31ebb5..3c3aee9333ef5e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -58,8 +58,7 @@ export const findThresholdSignals = async ({ top_hits: { sort: [ { - '@timestamp': { - // TODO: custom timestamp fields??? + [timestampOverride ?? '@timestamp']: { order: 'desc', }, }, From 7bbd12a8edcc60f20ca1c0139ac34dc0bc7ec4c3 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 15:42:12 +0000 Subject: [PATCH 08/21] Revert changes in signal creation --- .../signals/bulk_create_threshold_signals.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index a05151cffe35eb..2870b1ef5926a2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -53,12 +53,9 @@ interface FilterObject { const injectFirstMatch = ( hit: SignalSourceHit, - match: string | object | Record + match: object | Record ): Record | undefined => { if (match != null) { - if (typeof match === 'string') { - return { exists: get(match, hit._source) } as Record; - } for (const key of Object.keys(match)) { return { [key]: get(key, hit._source) } as Record; } @@ -88,8 +85,7 @@ const getNestedQueryFilters = ( (filtersObj.bool?.should && filtersObj.bool?.should[0] && (injectFirstMatch(hit, filtersObj.bool.should[0].match) || - injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase) || - injectFirstMatch(hit, filtersObj.bool.should[0].exists.field))) ?? + injectFirstMatch(hit, filtersObj.bool.should[0].match_phrase))) ?? {} ); } @@ -104,17 +100,11 @@ export const getThresholdSignalQueryFields = (hit: SignalSourceHit, filter: unkn return { ...acc, ...injectFirstMatch(hit, item.match_phrase) }; } - if ( - item.bool?.should && - (item.bool.should[0].match || - item.bool.should[0].match_phrase || - item.bool.should[0].exists) - ) { + if (item.bool?.should && (item.bool.should[0].match || item.bool.should[0].match_phrase)) { return { ...acc, ...(injectFirstMatch(hit, item.bool.should[0].match) || - injectFirstMatch(hit, item.bool.should[0].match_phrase) || - injectFirstMatch(hit, item.bool.should[0].exists.field)), + injectFirstMatch(hit, item.bool.should[0].match_phrase)), }; } From 960ed9b1e3ea1c5be594be4809304893fe9b683f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 16:57:53 +0000 Subject: [PATCH 09/21] Simplify query, return 10k buckets --- .../find_previous_threshold_signals.ts | 26 +++++++------------ .../signals/find_threshold_signals.ts | 1 + 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 41b108e096fc8c..51c6e0066b6473 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -40,23 +40,14 @@ export const findPreviousThresholdSignals = async ({ searchErrors: string[]; }> => { const aggregations = { - rule: { - filter: { - term: { - 'signal.rule.rule_id': ruleId, - }, + threshold: { + terms: { + field: bucketByField, }, aggs: { - threshold: { - terms: { - field: bucketByField, - }, - aggs: { - lastSignalTimestamp: { - max: { - field: 'signal.original_time', // timestamp of last event captured by bucket - }, - }, + lastSignalTimestamp: { + max: { + field: 'signal.original_time', // timestamp of last event captured by bucket }, }, }, @@ -64,10 +55,11 @@ export const findPreviousThresholdSignals = async ({ }; const filter = { - match_all: {}, + term: { + 'signal.rule.rule_id': ruleId, + }, }; - // TODO: paginate return singleSearchAfter({ aggregations, searchAfterSortId: undefined, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 3c3aee9333ef5e..8a83375f83a28d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -51,6 +51,7 @@ export const findThresholdSignals = async ({ terms: { field: threshold.field, min_doc_count: threshold.value, + size: 10000, // max 10k buckets }, aggs: { // Get the most recent hit per bucket From 859188624fa6e7888fd63eba73861981f0598072 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 17:37:03 +0000 Subject: [PATCH 10/21] Account for when threshold.field is not supplied --- .../signals/bulk_create_threshold_signals.ts | 35 ++++++++++++++----- .../signals/find_threshold_signals.ts | 2 +- .../signals/signal_rule_alert_type.ts | 1 + 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index 2870b1ef5926a2..edaaa345d8a69d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -8,7 +8,10 @@ import uuidv5 from 'uuid/v5'; import { reduce, get, isEmpty } from 'lodash/fp'; import set from 'set-value'; -import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas'; +import { + Threshold, + TimestampOverrideOrUndefined, +} from '../../../../common/detection_engine/schemas/common/schemas'; import { Logger } from '../../../../../../../src/core/server'; import { AlertServices } from '../../../../../alerts/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; @@ -30,6 +33,7 @@ interface BulkCreateThresholdSignalsParams { id: string; filter: unknown; signalsIndex: string; + timestampOverride: TimestampOverrideOrUndefined; name: string; createdAt: string; createdBy: string; @@ -123,9 +127,11 @@ const getTransformedHits = ( results: SignalSearchResponse, inputIndex: string, startedAt: Date, + logger: Logger, threshold: Threshold, ruleId: string, - filter: unknown + filter: unknown, + timestampOverride: TimestampOverrideOrUndefined ) => { if (isEmpty(threshold.field)) { const totalResults = @@ -135,11 +141,16 @@ const getTransformedHits = ( return []; } + const hit = results.hits.hits[0]; + if (hit == null) { + logger.warn(`No hits returned, but totalResults >= threshold.value (${threshold.value})`); + return []; + } + const source = { - '@timestamp': new Date().toISOString(), + '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), threshold_count: totalResults, - // TODO: how to get signal query fields for this case??? - // ...signalQueryFields, + ...getThresholdSignalQueryFields(hit, filter), }; return [ @@ -164,7 +175,7 @@ const getTransformedHits = ( } const source = { - '@timestamp': new Date().toISOString(), // TODO: use timestamp of latest event? + '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), threshold_count: docCount, ...getThresholdSignalQueryFields(hit, filter), }; @@ -186,16 +197,20 @@ export const transformThresholdResultsToEcs = ( inputIndex: string, startedAt: Date, filter: unknown, + logger: Logger, threshold: Threshold, - ruleId: string + ruleId: string, + timestampOverride: TimestampOverrideOrUndefined ): SignalSearchResponse => { const transformedHits = getTransformedHits( results, inputIndex, startedAt, + logger, threshold, ruleId, - filter + filter, + timestampOverride ); const thresholdResults = { ...results, @@ -219,8 +234,10 @@ export const bulkCreateThresholdSignals = async ( params.inputIndexPattern.join(','), params.startedAt, params.filter, + params.logger, params.ruleParams.threshold!, - params.ruleParams.ruleId + params.ruleParams.ruleId, + params.timestampOverride ); const buildRuleMessage = params.buildRuleMessage; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 3c3aee9333ef5e..d4489b4cadbf98 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -81,7 +81,7 @@ export const findThresholdSignals = async ({ services, logger, filter, - pageSize: 0, + pageSize: 1, buildRuleMessage, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 1d2b1c23f868f5..4eda9150e52f10 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -328,6 +328,7 @@ export const signalRulesAlertType = ({ id: alertId, inputIndexPattern: inputIndex, signalsIndex: outputIndex, + timestampOverride, startedAt, name, createdBy, From bf910ab14978f22c125c30aa364ea25b8ebb6256 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 18:13:34 +0000 Subject: [PATCH 11/21] Ensure we're getting the last event when threshold.field is not provided --- .../lib/detection_engine/signals/build_events_query.ts | 4 +++- .../detection_engine/signals/find_threshold_signals.ts | 1 + .../lib/detection_engine/signals/single_search_after.ts | 8 +++++++- .../lib/detection_engine/signals/threat_mapping/types.ts | 6 ++++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index 772645f06d7619..f77e26fa18919d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -13,6 +13,7 @@ interface BuildEventsSearchQuery { to: string; filter: unknown; size: number; + sortOrder?: SortOrderOrUndefined; searchAfterSortId: string | number | undefined; timestampOverride: TimestampOverrideOrUndefined; } @@ -25,6 +26,7 @@ export const buildEventsSearchQuery = ({ filter, size, searchAfterSortId, + sortOrder, timestampOverride, }: BuildEventsSearchQuery) => { const timestamp = timestampOverride ?? '@timestamp'; @@ -108,7 +110,7 @@ export const buildEventsSearchQuery = ({ sort: [ { [timestamp]: { - order: 'asc', + order: sortOrder ?? 'asc', }, }, ], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index d4489b4cadbf98..01e4812b9c8bfb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -82,6 +82,7 @@ export const findThresholdSignals = async ({ logger, filter, pageSize: 1, + sortOrder: 'desc', buildRuleMessage, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts index 3b89a2d79c0d04..23ef9fcea8e530 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_search_after.ts @@ -11,7 +11,10 @@ import { SignalSearchResponse } from './types'; import { BuildRuleMessage } from './rule_messages'; import { buildEventsSearchQuery } from './build_events_query'; import { createErrorsFromShard, makeFloatString } from './utils'; -import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { + SortOrderOrUndefined, + TimestampOverrideOrUndefined, +} from '../../../../common/detection_engine/schemas/common/schemas'; interface SingleSearchAfterParams { aggregations?: unknown; @@ -22,6 +25,7 @@ interface SingleSearchAfterParams { services: AlertServices; logger: Logger; pageSize: number; + sortOrder?: SortOrderOrUndefined; filter: unknown; timestampOverride: TimestampOverrideOrUndefined; buildRuleMessage: BuildRuleMessage; @@ -38,6 +42,7 @@ export const singleSearchAfter = async ({ filter, logger, pageSize, + sortOrder, timestampOverride, buildRuleMessage, }: SingleSearchAfterParams): Promise<{ @@ -53,6 +58,7 @@ export const singleSearchAfter = async ({ to, filter, size: pageSize, + sortOrder, searchAfterSortId, timestampOverride, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 2e32a4e682403f..faad51e4751e84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -28,6 +28,8 @@ import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { SearchAfterAndBulkCreateReturnType } from '../types'; +export type SortOrderOrUndefined = 'asc' | 'desc' | undefined; + export interface CreateThreatSignalsOptions { threatMapping: ThreatMapping; query: string; @@ -146,7 +148,7 @@ export interface GetThreatListOptions { perPage?: number; searchAfter: string[] | undefined; sortField: string | undefined; - sortOrder: 'asc' | 'desc' | undefined; + sortOrder: SortOrderOrUndefined; threatFilters: PartialFilter[]; exceptionItems: ExceptionListItemSchema[]; listClient: ListClient; @@ -165,7 +167,7 @@ export interface ThreatListCountOptions { export interface GetSortWithTieBreakerOptions { sortField: string | undefined; - sortOrder: 'asc' | 'desc' | undefined; + sortOrder: SortOrderOrUndefined; index: string[]; listItemIndex: string; } From 83f81cb839acdded0fe5a98c106d68b9063d6a1c Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 19:03:51 +0000 Subject: [PATCH 12/21] Add missing import --- .../lib/detection_engine/signals/build_events_query.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts index f77e26fa18919d..beca56770a9cad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_events_query.ts @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { + SortOrderOrUndefined, + TimestampOverrideOrUndefined, +} from '../../../../common/detection_engine/schemas/common/schemas'; interface BuildEventsSearchQuery { aggregations?: unknown; From 8d6c81d3ffa5a84b70c37c5e227016a076b808ea Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 10 Nov 2020 20:49:35 +0000 Subject: [PATCH 13/21] Handle case where threshold field not supplied --- .../detection_engine/signals/find_previous_threshold_signals.ts | 2 +- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 51c6e0066b6473..1c3453ad402fcf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -42,7 +42,7 @@ export const findPreviousThresholdSignals = async ({ const aggregations = { threshold: { terms: { - field: bucketByField, + field: bucketByField ?? 'signal.rule.rule_id', }, aggs: { lastSignalTimestamp: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index c9d30065512916..fceac3c0a42589 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -324,7 +324,7 @@ export const signalRulesAlertType = ({ must: [ { term: { - [threshold.field]: bucket.key, + [threshold.field ?? 'signal.rule.rule_id']: bucket.key, }, }, { From a087823852b40202dec2f3e6693f3c11595c8898 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 11 Nov 2020 03:29:02 +0000 Subject: [PATCH 14/21] Fix type errors --- .../signals/signal_rule_alert_type.ts | 42 ++++++++++--------- .../lib/detection_engine/signals/types.ts | 6 +++ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index fceac3c0a42589..c376c9e7bda874 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,6 +8,7 @@ import { Logger, KibanaRequest } from 'src/core/server'; +import { Filter } from 'src/plugins/data/common'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -29,6 +30,7 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, + ThresholdQueryBucket, } from './types'; import { getGapBetweenRuns, @@ -316,30 +318,32 @@ export const signalRulesAlertType = ({ buildRuleMessage, }); - previousSignals.aggregations.rule.threshold.buckets.forEach((bucket: object) => { - esFilter.bool.filter.push({ - bool: { - must_not: { - bool: { - must: [ - { - term: { - [threshold.field ?? 'signal.rule.rule_id']: bucket.key, + previousSignals.aggregations.rule.threshold.buckets.forEach( + (bucket: ThresholdQueryBucket) => { + esFilter.bool.filter.push(({ + bool: { + must_not: { + bool: { + must: [ + { + term: { + [threshold.field ?? 'signal.rule.rule_id']: bucket.key, + }, }, - }, - { - range: { - '@timestamp': { - lte: bucket.lastSignalTimestamp.value_as_string, + { + range: { + '@timestamp': { + lte: bucket.lastSignalTimestamp.value_as_string, + }, }, }, - }, - ], + ], + }, }, }, - }, - }); - }); + } as unknown) as Filter); + } + ); const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index cda3c97c085316..6b1556fcc7c667 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -239,3 +239,9 @@ export interface SearchAfterAndBulkCreateReturnType { export interface ThresholdAggregationBucket extends TermAggregationBucket { top_threshold_hits: BaseSearchResponse; } + +export interface ThresholdQueryBucket extends TermAggregationBucket { + lastSignalTimestamp: { + value_as_string: string; + }; +} From ae24022950f4b14f7ac6f808b9f97574cd5efcb9 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 12 Nov 2020 18:05:57 +0000 Subject: [PATCH 15/21] Handle non-ECS fields --- .../routes/index/signals_mapping.json | 11 ++++- .../signals/build_bulk_body.ts | 2 + .../detection_engine/signals/build_signal.ts | 3 +- .../signals/bulk_create_threshold_signals.ts | 12 +++++- .../find_previous_threshold_signals.ts | 17 ++++++-- .../signals/signal_rule_alert_type.ts | 40 +++++++++---------- 6 files changed, 56 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 4e9477f3f2f73b..6f0a1c8da40c60 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -334,8 +334,15 @@ "status": { "type": "keyword" }, - "threshold_count": { - "type": "float" + "threshold_bucket": { + "properties": { + "count": { + "type": "float" + }, + "match_value": { + "type": "keyword" + } + } }, "depth": { "type": "integer" diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index a704d076880bf3..a10d624dd81639 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -71,6 +71,8 @@ export const buildBulkBody = ({ ...buildSignal([doc], rule), ...additionalSignalFields(doc), }; + delete doc._source.threshold_count; // TODO: remove/deprecate + delete doc._source.threshold_bucket; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index b36a1cbb4a6b3e..0c53693bb5233e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -98,7 +98,8 @@ export const additionalSignalFields = (doc: BaseSignalHit) => { parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, - threshold_count: doc._source.threshold_count ?? undefined, + threshold_count: doc._source.threshold_count ?? undefined, // TODO: remove/deprecate + threshold_bucket: doc._source.threshold_bucket, original_signal: doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index edaaa345d8a69d..aafc36f299912f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -149,7 +149,11 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: totalResults, + threshold_count: totalResults, // TODO: remove/deprecate this + threshold_bucket: { + count: totalResults, + match_value: ruleId, + }, ...getThresholdSignalQueryFields(hit, filter), }; @@ -176,7 +180,11 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: docCount, + threshold_count: docCount, // TODO: remove/deprecate this + threshold_bucket: { + count: docCount, + match_value: get(threshold.field, hit._source), + }, ...getThresholdSignalQueryFields(hit, filter), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 1c3453ad402fcf..75b8bdfada12f7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -42,7 +42,7 @@ export const findPreviousThresholdSignals = async ({ const aggregations = { threshold: { terms: { - field: bucketByField ?? 'signal.rule.rule_id', + field: 'signal.threshold_bucket.match_value', }, aggs: { lastSignalTimestamp: { @@ -55,8 +55,19 @@ export const findPreviousThresholdSignals = async ({ }; const filter = { - term: { - 'signal.rule.rule_id': ruleId, + bool: { + must: [ + { + term: { + 'signal.rule.rule_id': ruleId, + }, + }, + { + term: { + 'signal.rule.threshold.field': bucketByField, + }, + }, + ], }, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index c376c9e7bda874..df888c27025d94 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -318,32 +318,30 @@ export const signalRulesAlertType = ({ buildRuleMessage, }); - previousSignals.aggregations.rule.threshold.buckets.forEach( - (bucket: ThresholdQueryBucket) => { - esFilter.bool.filter.push(({ - bool: { - must_not: { - bool: { - must: [ - { - term: { - [threshold.field ?? 'signal.rule.rule_id']: bucket.key, - }, + previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { + esFilter.bool.filter.push(({ + bool: { + must_not: { + bool: { + must: [ + { + term: { + [threshold.field ?? 'signal.rule.rule_id']: bucket.key, }, - { - range: { - '@timestamp': { - lte: bucket.lastSignalTimestamp.value_as_string, - }, + }, + { + range: { + '@timestamp': { + lte: bucket.lastSignalTimestamp.value_as_string, }, }, - ], - }, + }, + ], }, }, - } as unknown) as Filter); - } - ); + }, + } as unknown) as Filter); + }); const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, From 2c8dcfa6c76a17f95ea832f11dcaaa78479d816f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 12 Nov 2020 20:35:56 +0000 Subject: [PATCH 16/21] Regorganize --- .../routes/index/signals_mapping.json | 6 +++--- .../lib/detection_engine/signals/build_bulk_body.ts | 3 +-- .../lib/detection_engine/signals/build_signal.ts | 3 +-- .../signals/bulk_create_threshold_signals.ts | 10 ++++------ .../signals/find_previous_threshold_signals.ts | 2 +- .../server/lib/detection_engine/signals/types.ts | 5 ++++- .../endpoint/resolver/signals/mappings.json | 11 +++++++++-- .../es_archives/export_rule/mappings.json | 11 +++++++++-- 8 files changed, 32 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 6f0a1c8da40c60..cd1d655e45a241 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -334,12 +334,12 @@ "status": { "type": "keyword" }, - "threshold_bucket": { + "threshold_result": { "properties": { "count": { - "type": "float" + "type": "long" }, - "match_value": { + "value": { "type": "keyword" } } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index a10d624dd81639..8644306056e090 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -71,8 +71,7 @@ export const buildBulkBody = ({ ...buildSignal([doc], rule), ...additionalSignalFields(doc), }; - delete doc._source.threshold_count; // TODO: remove/deprecate - delete doc._source.threshold_bucket; + delete doc._source.threshold_result; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 0c53693bb5233e..381984f6b0ec73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -98,8 +98,7 @@ export const additionalSignalFields = (doc: BaseSignalHit) => { parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, - threshold_count: doc._source.threshold_count ?? undefined, // TODO: remove/deprecate - threshold_bucket: doc._source.threshold_bucket, + threshold_result: doc._source.threshold_result, original_signal: doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index aafc36f299912f..a98aae4ec8107b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -149,10 +149,9 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: totalResults, // TODO: remove/deprecate this - threshold_bucket: { + threshold_result: { count: totalResults, - match_value: ruleId, + value: ruleId, }, ...getThresholdSignalQueryFields(hit, filter), }; @@ -180,10 +179,9 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: docCount, // TODO: remove/deprecate this - threshold_bucket: { + threshold_result: { count: docCount, - match_value: get(threshold.field, hit._source), + value: get(threshold.field, hit._source), }, ...getThresholdSignalQueryFields(hit, filter), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts index 75b8bdfada12f7..960693bc703d67 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -42,7 +42,7 @@ export const findPreviousThresholdSignals = async ({ const aggregations = { threshold: { terms: { - field: 'signal.threshold_bucket.match_value', + field: 'signal.threshold_result.value', }, aggs: { lastSignalTimestamp: { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 6b1556fcc7c667..471071a8a7abb8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -156,7 +156,10 @@ export interface Signal { original_time?: string; original_event?: SearchTypes; status: Status; - threshold_count?: SearchTypes; + threshold_result?: { + count: SearchTypes; + value: string; + }; original_signal?: SearchTypes; depth: number; } diff --git a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json index ad77961a41445f..47bcdcdea5abff 100644 --- a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json @@ -2580,8 +2580,15 @@ "status": { "type": "keyword" }, - "threshold_count": { - "type": "float" + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } }, diff --git a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json index 5eec03ca3d11a8..a13f96b247ba4a 100644 --- a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json @@ -5126,8 +5126,15 @@ "status": { "type": "keyword" }, - "threshold_count": { - "type": "float" + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } }, From 23e4ec3cbb9cc8ed97cc7e6be0bcb80c51324c3f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 23 Nov 2020 23:06:00 +0000 Subject: [PATCH 17/21] Address comments --- .../lib/detection_engine/routes/index/get_signals_template.ts | 2 +- .../es_archives/endpoint/resolver/signals/mappings.json | 3 +++ .../es_archives/export_rule/mappings.json | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index d1a9b701b2c9d8..8ea1faa84cfba5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,7 +7,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 2; +export const SIGNALS_TEMPLATE_VERSION = 3; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json index 47bcdcdea5abff..3c6042e8efd240 100644 --- a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json @@ -2580,6 +2580,9 @@ "status": { "type": "keyword" }, + "threshold_count": { + "type": "float" + }, "threshold_result": { "properties": { "count": { diff --git a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json index 05a05a72dec892..f701f811b244b7 100644 --- a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json @@ -5129,6 +5129,9 @@ "status": { "type": "keyword" }, + "threshold_count": { + "type": "float" + }, "threshold_result": { "properties": { "count": { From 1e8d122a165a2459e8123251440d994943bf3af5 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 24 Nov 2020 00:34:32 +0000 Subject: [PATCH 18/21] Fix type error --- .../server/lib/detection_engine/signals/types.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 471071a8a7abb8..9e81797b147315 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -46,6 +46,11 @@ export interface SignalsStatusParams { status: Status; } +export interface ThresholdResult { + count: number; + value: string; +} + export interface SignalSource { [key: string]: SearchTypes; // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not @@ -67,6 +72,7 @@ export interface SignalSource { // signal.depth doesn't exist on pre-7.10 signals depth?: number; }; + threshold_result?: ThresholdResult; } export interface BulkItem { @@ -156,10 +162,7 @@ export interface Signal { original_time?: string; original_event?: SearchTypes; status: Status; - threshold_result?: { - count: SearchTypes; - value: string; - }; + threshold_result?: ThresholdResult; original_signal?: SearchTypes; depth: number; } From 3a54495332396b8eace1a582cd7a6113c8baff54 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 24 Nov 2020 15:26:02 +0000 Subject: [PATCH 19/21] Add unit test for buildBulkBody on threshold results --- .../signals/build_bulk_body.test.ts | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ad060a9304e848..cd61fbcfd0fc7c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -123,6 +123,115 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); + test('bulk body builds well-defined body with threshold results', () => { + const sampleParams = sampleRuleAlertParams(); + const baseDoc = sampleDocNoSortId(); + const doc: SignalSourceHit = { + ...baseDoc, + _source: { + ...baseDoc._source, + threshold_result: { + count: 5, + value: 'abcd', + }, + }, + }; + delete doc._source.source; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete fakeSignalSourceHit['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + throttle: 'no_actions', + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + exceptions_list: getListArrayMock(), + }, + threshold_result: { + count: 5, + value: 'abcd', + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + test('bulk body builds original_event if it exists on the event to begin with', () => { const sampleParams = sampleRuleAlertParams(); const doc = sampleDocNoSortId(); From 80a582ac08515d635638375d05e53caaa05085d9 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 25 Nov 2020 19:50:19 +0000 Subject: [PATCH 20/21] Add threshold_count back to mapping (and deprecate) --- .../lib/detection_engine/routes/index/signals_mapping.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index cd1d655e45a241..96868e62ea978c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -334,6 +334,9 @@ "status": { "type": "keyword" }, + "threshold_count": { + "type": "float" + }, "threshold_result": { "properties": { "count": { From 2c61c2623ecfcae11eb359ef65983891218e874f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 25 Nov 2020 20:16:44 +0000 Subject: [PATCH 21/21] Timestamp fixes --- .../server/lib/detection_engine/signals/build_signal.ts | 2 +- .../lib/detection_engine/signals/signal_rule_alert_type.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 381984f6b0ec73..9cf2462526cfce 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -96,7 +96,7 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(removeClashes(doc)), - original_time: doc._source['@timestamp'], + original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. original_event: doc._source.event ?? undefined, threshold_result: doc._source.threshold_result, original_signal: diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 201ef0542ee155..f0b1825c7cc99c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -331,7 +331,7 @@ export const signalRulesAlertType = ({ }, { range: { - '@timestamp': { + [timestampOverride ?? '@timestamp']: { lte: bucket.lastSignalTimestamp.value_as_string, }, },