From 46513acc8ec5b24158d4f0b3c6c6c5db0f8d1ef7 Mon Sep 17 00:00:00 2001 From: Steph Milovic Date: Thu, 25 Feb 2021 20:10:53 -0700 Subject: [PATCH] WIP events/all --- .../search_strategy/helpers/to_array.ts | 35 +- .../search_strategy/timeline/eql/helpers.ts | 78 +++-- .../search_strategy/timeline/eql/index.ts | 2 +- .../factory/events/all/helpers.test.ts | 105 +++--- .../timeline/factory/events/all/helpers.ts | 76 ++-- .../timeline/factory/events/all/index.ts | 7 +- .../factory/events/details/helpers.test.ts | 321 +---------------- .../factory/events/details/helpers.ts | 55 +-- .../timeline/factory/events/details/index.ts | 15 +- .../timeline/factory/events/mocks.ts | 329 ++++++++++++++++++ 10 files changed, 571 insertions(+), 452 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts diff --git a/x-pack/plugins/security_solution/server/search_strategy/helpers/to_array.ts b/x-pack/plugins/security_solution/server/search_strategy/helpers/to_array.ts index aa762af2b61627..fbb2b8d48a250d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/helpers/to_array.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/helpers/to_array.ts @@ -7,7 +7,40 @@ export const toArray = (value: T | T[] | null): T[] => Array.isArray(value) ? value : value == null ? [] : [value]; - +export const toStringArray = (value: T | T[] | null): string[] => { + if (Array.isArray(value)) { + return value.reduce((acc, v) => { + if (v != null) { + switch (typeof v) { + case 'number': + case 'boolean': + return [...acc, v.toString()]; + case 'object': + try { + return [...acc, JSON.stringify(v)]; + } catch { + return [...acc, 'Invalid Object']; + } + case 'string': + return [...acc, v]; + default: + return [...acc, `${v}`]; + } + } + return acc; + }, []); + } else if (value == null) { + return []; + } else if (!Array.isArray(value) && typeof value === 'object') { + try { + return [JSON.stringify(value)]; + } catch { + return ['Invalid Object']; + } + } else { + return [`${value}`]; + } +}; export const toObjectArrayOfStrings = ( value: T | T[] | null ): Array<{ diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts index bdd3195b3b756d..b007307412e955 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/helpers.ts @@ -7,7 +7,7 @@ import { EqlSearchStrategyResponse } from '../../../../../data_enhanced/common'; import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../common/constants'; -import { EqlSearchResponse } from '../../../../common/detection_engine/types'; +import { EqlSearchResponse, EqlSequence } from '../../../../common/detection_engine/types'; import { EventHit, TimelineEdges } from '../../../../common/search_strategy'; import { TimelineEqlRequestOptions, @@ -56,51 +56,53 @@ export const buildEqlDsl = (options: TimelineEqlRequestOptions): Record>, fieldRequested: string[]) => + sequences.reduce>(async (acc, sequence, sequenceIndex) => { + const sequenceParentId = sequence.events[0]?._id ?? null; + const data = await acc; + const allData = await Promise.all( + sequence.events.map(async (event, eventIndex) => { + const item = await formatTimelineData( + fieldRequested, + TIMELINE_EVENTS_FIELDS, + event as EventHit + ); + return Promise.resolve({ + ...item, + node: { + ...item.node, + ecs: { + ...item.node.ecs, + ...(sequenceParentId != null + ? { + eql: { + parentId: sequenceParentId, + sequenceNumber: `${sequenceIndex}-${eventIndex}`, + }, + } + : {}), + }, + }, + }); + }) + ); + return Promise.resolve([...data, ...allData]); + }, Promise.resolve([])); -export const parseEqlResponse = ( +export const parseEqlResponse = async ( options: TimelineEqlRequestOptions, response: EqlSearchStrategyResponse> ): Promise => { const { activePage, querySize } = options.pagination; - // const totalCount = response.rawResponse?.body?.hits?.total?.value ?? 0; let edges: TimelineEdges[] = []; + if (response.rawResponse.body.hits.sequences !== undefined) { - edges = response.rawResponse.body.hits.sequences.reduce( - (data, sequence, sequenceIndex) => { - const sequenceParentId = sequence.events[0]?._id ?? null; - return [ - ...data, - ...sequence.events.map((event, eventIndex) => { - const item = formatTimelineData( - options.fieldRequested, - TIMELINE_EVENTS_FIELDS, - event as EventHit - ); - return { - ...item, - node: { - ...item.node, - ecs: { - ...item.node.ecs, - ...(sequenceParentId != null - ? { - eql: { - parentId: sequenceParentId, - sequenceNumber: `${sequenceIndex}-${eventIndex}`, - }, - } - : {}), - }, - }, - }; - }), - ]; - }, - [] - ); + edges = await parseSequences(response.rawResponse.body.hits.sequences, options.fieldRequested); } else if (response.rawResponse.body.hits.events !== undefined) { - edges = response.rawResponse.body.hits.events.map((event) => - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) + edges = await Promise.all( + response.rawResponse.body.hits.events.map(async (event) => + formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, event as EventHit) + ) ); } diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts index cf7877e987acea..249f5582d2a398 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/eql/index.ts @@ -38,7 +38,7 @@ export const securitySolutionTimelineEqlSearchStrategyProvider = ( }, }; }), - mergeMap((esSearchRes) => + mergeMap(async (esSearchRes) => parseEqlResponse( request, (esSearchRes as unknown) as EqlSearchStrategyResponse> diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts index 10bb606dc2387e..0646333615b2f9 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.test.ts @@ -8,54 +8,23 @@ import { EventHit } from '../../../../../../common/search_strategy'; import { TIMELINE_EVENTS_FIELDS } from './constants'; import { formatTimelineData } from './helpers'; +import { eventHit } from '../mocks'; describe('#formatTimelineData', () => { - it('happy path', () => { - const response: EventHit = { - _index: 'auditbeat-7.8.0-2020.11.05-000003', - _id: 'tkCt1nUBaEgqnrVSZ8R_', - _score: 0, - _type: '', - fields: { - 'event.category': ['process'], - 'process.ppid': [3977], - 'user.name': ['jenkins'], - 'process.args': ['go', 'vet', './...'], - message: ['Process go (PID: 4313) by user jenkins STARTED'], - 'process.pid': [4313], - 'process.working_directory': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], - 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - 'process.name': ['go'], - 'event.action': ['process_started'], - 'agent.type': ['auditbeat'], - '@timestamp': ['2020-11-17T14:48:08.922Z'], - 'event.module': ['system'], - 'event.type': ['start'], - 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - 'host.os.family': ['debian'], - 'event.kind': ['event'], - 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], - 'event.dataset': ['process'], - 'process.executable': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - }, - _source: {}, - sort: ['1605624488922', 'beats-ci-immutable-ubuntu-1804-1605624279743236239'], - aggregations: {}, - }; - - expect( - formatTimelineData( - ['@timestamp', 'host.name', 'destination.ip', 'source.ip'], - TIMELINE_EVENTS_FIELDS, - response - ) - ).toEqual({ + it('happy path', async () => { + const res = await formatTimelineData( + [ + '@timestamp', + 'host.name', + 'destination.ip', + 'source.ip', + 'source.geo.location', + 'threat.indicator', + ], + TIMELINE_EVENTS_FIELDS, + eventHit + ); + expect(res).toEqual({ cursor: { tiebreaker: 'beats-ci-immutable-ubuntu-1804-1605624279743236239', value: '1605624488922', @@ -72,6 +41,50 @@ describe('#formatTimelineData', () => { field: 'host.name', value: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], }, + { + field: 'source.geo.location', + value: [`{"lon":118.7778,"lat":32.0617}`], + }, + { + field: 'threat.indicator.matched.field', + value: ['matched_field', 'matched_field_2'], + }, + { + field: 'threat.indicator.first_seen', + value: ['2021-02-22T17:29:25.195Z'], + }, + { + field: 'threat.indicator.provider', + value: ['yourself', 'other_you'], + }, + { + field: 'threat.indicator.type', + value: ['custom'], + }, + { + field: 'threat.indicator.matched.atomic', + value: ['matched_atomic', 'matched_atomic_2'], + }, + { + field: 'threat.indicator.lazer.great.field', + value: ['grrrrr', 'grrrrr_2'], + }, + { + field: 'threat.indicator.lazer.great.field.wowoe.fooooo', + value: ['grrrrr'], + }, + { + field: 'threat.indicator.lazer.great.field.astring', + value: ['cool'], + }, + { + field: 'threat.indicator.lazer.great.field.aNumber', + value: ['1'], + }, + { + field: 'threat.indicator.lazer.great.field.neat', + value: ['true'], + }, ], ecs: { '@timestamp': ['2020-11-17T14:48:08.922Z'], diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts index 09a833c912d5b0..9551f1d6643749 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/helpers.ts @@ -6,9 +6,13 @@ */ import { get, has, merge, uniq } from 'lodash/fp'; -import { EventHit, TimelineEdges } from '../../../../../../common/search_strategy'; -import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; -import { formatGeoLocation, isGeoField } from '../details/helpers'; +import { + EventHit, + TimelineEdges, + TimelineNonEcsData, +} from '../../../../../../common/search_strategy'; +import { toStringArray } from '../../../../helpers/to_array'; +import { getDataSafety, getDataFromFieldsHits } from '../details/helpers'; const getTimestamp = (hit: EventHit): string => { if (hit.fields && hit.fields['@timestamp']) { @@ -19,13 +23,14 @@ const getTimestamp = (hit: EventHit): string => { return ''; }; -export const formatTimelineData = ( +export const formatTimelineData = async ( dataFields: readonly string[], ecsFields: readonly string[], hit: EventHit ) => - uniq([...ecsFields, ...dataFields]).reduce( - (flattenedFields, fieldName) => { + uniq([...ecsFields, ...dataFields]).reduce>( + async (acc, fieldName) => { + const flattenedFields: TimelineEdges = await acc; flattenedFields.node._id = hit._id; flattenedFields.node._index = hit._index; flattenedFields.node.ecs._id = hit._id; @@ -35,27 +40,51 @@ export const formatTimelineData = ( flattenedFields.cursor.value = hit.sort[0]; flattenedFields.cursor.tiebreaker = hit.sort[1]; } - return mergeTimelineFieldsWithHit(fieldName, flattenedFields, hit, dataFields, ecsFields); + const waitForIt = await mergeTimelineFieldsWithHit( + fieldName, + flattenedFields, + hit, + dataFields, + ecsFields + ); + return Promise.resolve(waitForIt); }, - { + Promise.resolve({ node: { ecs: { _id: '' }, data: [], _id: '', _index: '' }, cursor: { value: '', tiebreaker: null, }, - } + }) ); const specialFields = ['_id', '_index', '_type', '_score']; -const mergeTimelineFieldsWithHit = ( +const getValuesFromFields = async ( + fieldName: string, + hit: EventHit +): Promise => { + if (specialFields.includes(fieldName)) { + return [{ field: fieldName, value: toStringArray(get(fieldName, hit)) }]; + } + const fieldToEval = has(fieldName, hit._source) + ? get(fieldName, hit._source) + : hit.fields[fieldName]; + const formattedData = await getDataSafety(getDataFromFieldsHits, { + [fieldName]: fieldToEval, + }); + return formattedData.map(({ field, values }) => ({ field, value: values })); +}; + +const mergeTimelineFieldsWithHit = async ( fieldName: string, flattenedFields: T, - hit: { _source: {}; fields: Record }, + hit: EventHit, dataFields: readonly string[], ecsFields: readonly string[] ) => { if (fieldName != null || dataFields.includes(fieldName)) { + console.log('fieldName', fieldName); if ( has(fieldName, hit._source) || has(fieldName, hit.fields) || @@ -65,19 +94,7 @@ const mergeTimelineFieldsWithHit = ( node: { ...get('node', flattenedFields), data: dataFields.includes(fieldName) - ? [ - ...get('node.data', flattenedFields), - { - field: fieldName, - value: specialFields.includes(fieldName) - ? toObjectArrayOfStrings(get(fieldName, hit)).map(({ str }) => str) - : isGeoField(fieldName) - ? formatGeoLocation(hit.fields[fieldName]) - : has(fieldName, hit._source) - ? toObjectArrayOfStrings(get(fieldName, hit._source)).map(({ str }) => str) - : toObjectArrayOfStrings(hit.fields[fieldName]).map(({ str }) => str), - }, - ] + ? [...get('node.data', flattenedFields), ...(await getValuesFromFields(fieldName, hit))] : get('node.data', flattenedFields), ecs: ecsFields.includes(fieldName) ? { @@ -86,18 +103,25 @@ const mergeTimelineFieldsWithHit = ( ...fieldName.split('.').reduceRight( // @ts-expect-error (obj, next) => ({ [next]: obj }), - toObjectArrayOfStrings( + toStringArray( has(fieldName, hit._source) ? get(fieldName, hit._source) : hit.fields[fieldName] - ).map(({ str }) => str) + ) ), } : get('node.ecs', flattenedFields), }, }; + console.log('DATA!!!', { + flattenedFields: flattenedFields.node.data, + objectWithProperty: objectWithProperty.node.data, + }); return merge(flattenedFields, objectWithProperty); } else { + if (fieldName.includes('threat.indicator')) { + console.log('flattenedFields!!!', flattenedFields.node.data); + } return flattenedFields; } } else { diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts index 93985baed770e9..c57f5ce0467fdb 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/all/index.ts @@ -40,9 +40,12 @@ export const timelineEventsAll: SecuritySolutionTimelineFactory - formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, hit as EventHit) + const edges: TimelineEdges[] = await Promise.all( + hits.map((hit) => + formatTimelineData(options.fieldRequested, TIMELINE_EVENTS_FIELDS, hit as EventHit) + ) ); + console.log('edges??', edges); const inspect = { dsl: [inspectStringifyObject(buildTimelineEventsAllQuery(queryOptions))], }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts index 6406f6314cb04c..dc3efc6909c634 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.test.ts @@ -6,317 +6,12 @@ */ import { EventHit, EventSource } from '../../../../../../common/search_strategy'; -import { - getDataFromFieldsHits, - getDataFromSourceHits, - getDataFromFieldsHitsSafety, -} from './helpers'; +import { getDataFromFieldsHits, getDataFromSourceHits, getDataSafety } from './helpers'; +import { eventDetailsFormattedFields, eventHit } from '../mocks'; describe('Events Details Helpers', () => { - const fields: EventHit['fields'] = { - 'event.category': ['process'], - 'process.ppid': [3977], - 'user.name': ['jenkins'], - 'process.args': ['go', 'vet', './...'], - message: ['Process go (PID: 4313) by user jenkins STARTED'], - 'process.pid': [4313], - 'process.working_directory': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], - 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - 'process.name': ['go'], - 'event.action': ['process_started'], - 'agent.type': ['auditbeat'], - '@timestamp': ['2020-11-17T14:48:08.922Z'], - 'event.module': ['system'], - 'event.type': ['start'], - 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - 'host.os.family': ['debian'], - 'event.kind': ['event'], - 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], - 'event.dataset': ['process'], - 'process.executable': [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - 'threat.indicator': [ - { - 'matched.field': ['matched_field'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['yourself'], - type: ['custom'], - 'matched.atomic': ['matched_atomic'], - lazer: [ - { - 'great.field': ['grrrrr'], - }, - { - 'great.field': ['grrrrr_2'], - }, - ], - }, - { - 'matched.field': ['matched_field_2'], - first_seen: ['2021-02-22T17:29:25.195Z'], - provider: ['other_you'], - type: ['custom'], - 'matched.atomic': ['matched_atomic_2'], - lazer: [ - { - 'great.field': [ - { - wowoe: [ - { - fooooo: ['grrrrr'], - }, - ], - astring: 'cool', - aNumber: 1, - anObject: { - neat: true, - }, - }, - ], - }, - ], - }, - ], - }; - const resultFields = [ - { - category: 'event', - field: 'event.category', - isObjectArray: false, - originalValue: ['process'], - values: ['process'], - }, - { - category: 'process', - field: 'process.ppid', - isObjectArray: false, - originalValue: ['3977'], - values: ['3977'], - }, - { - category: 'user', - field: 'user.name', - isObjectArray: false, - originalValue: ['jenkins'], - values: ['jenkins'], - }, - { - category: 'process', - field: 'process.args', - isObjectArray: false, - originalValue: ['go', 'vet', './...'], - values: ['go', 'vet', './...'], - }, - { - category: 'base', - field: 'message', - isObjectArray: false, - originalValue: ['Process go (PID: 4313) by user jenkins STARTED'], - values: ['Process go (PID: 4313) by user jenkins STARTED'], - }, - { - category: 'process', - field: 'process.pid', - isObjectArray: false, - originalValue: ['4313'], - values: ['4313'], - }, - { - category: 'process', - field: 'process.working_directory', - isObjectArray: false, - originalValue: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - values: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', - ], - }, - { - category: 'process', - field: 'process.entity_id', - isObjectArray: false, - originalValue: ['Z59cIkAAIw8ZoK0H'], - values: ['Z59cIkAAIw8ZoK0H'], - }, - { - category: 'host', - field: 'host.ip', - isObjectArray: false, - originalValue: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - values: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], - }, - { - category: 'process', - field: 'process.name', - isObjectArray: false, - originalValue: ['go'], - values: ['go'], - }, - { - category: 'event', - field: 'event.action', - isObjectArray: false, - originalValue: ['process_started'], - values: ['process_started'], - }, - { - category: 'agent', - field: 'agent.type', - isObjectArray: false, - originalValue: ['auditbeat'], - values: ['auditbeat'], - }, - { - category: 'base', - field: '@timestamp', - isObjectArray: false, - originalValue: ['2020-11-17T14:48:08.922Z'], - values: ['2020-11-17T14:48:08.922Z'], - }, - { - category: 'event', - field: 'event.module', - isObjectArray: false, - originalValue: ['system'], - values: ['system'], - }, - { - category: 'event', - field: 'event.type', - isObjectArray: false, - originalValue: ['start'], - values: ['start'], - }, - { - category: 'host', - field: 'host.name', - isObjectArray: false, - originalValue: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - values: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], - }, - { - category: 'process', - field: 'process.hash.sha1', - isObjectArray: false, - originalValue: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - values: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], - }, - { - category: 'host', - field: 'host.os.family', - isObjectArray: false, - originalValue: ['debian'], - values: ['debian'], - }, - { - category: 'event', - field: 'event.kind', - isObjectArray: false, - originalValue: ['event'], - values: ['event'], - }, - { - category: 'host', - field: 'host.id', - isObjectArray: false, - originalValue: ['e59991e835905c65ed3e455b33e13bd6'], - values: ['e59991e835905c65ed3e455b33e13bd6'], - }, - { - category: 'event', - field: 'event.dataset', - isObjectArray: false, - originalValue: ['process'], - values: ['process'], - }, - { - category: 'process', - field: 'process.executable', - isObjectArray: false, - originalValue: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - values: [ - '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', - ], - }, - { - category: 'threat', - field: 'threat.indicator.matched.field', - values: ['matched_field', 'matched_field_2'], - originalValue: ['matched_field', 'matched_field_2'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.first_seen', - values: ['2021-02-22T17:29:25.195Z'], - originalValue: ['2021-02-22T17:29:25.195Z'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.provider', - values: ['yourself', 'other_you'], - originalValue: ['yourself', 'other_you'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.type', - values: ['custom'], - originalValue: ['custom'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.matched.atomic', - values: ['matched_atomic', 'matched_atomic_2'], - originalValue: ['matched_atomic', 'matched_atomic_2'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.lazer.great.field', - values: ['grrrrr', 'grrrrr_2'], - originalValue: ['grrrrr', 'grrrrr_2'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.lazer.great.field.wowoe.fooooo', - values: ['grrrrr'], - originalValue: ['grrrrr'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.lazer.great.field.astring', - values: ['cool'], - originalValue: ['cool'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.lazer.great.field.aNumber', - values: ['1'], - originalValue: ['1'], - isObjectArray: false, - }, - { - category: 'threat', - field: 'threat.indicator.lazer.great.field.neat', - values: ['true'], - originalValue: ['true'], - isObjectArray: false, - }, - ]; + const fields: EventHit['fields'] = eventHit.fields; + const resultFields = eventDetailsFormattedFields; describe('#getDataFromFieldsHits', () => { it('happy path', () => { const result = getDataFromFieldsHits(fields); @@ -437,10 +132,6 @@ describe('Events Details Helpers', () => { expect(result).toEqual(whackResultFields); }); }); - it('#getDataFromFieldsHitsSafety', async () => { - const result = await getDataFromFieldsHitsSafety(fields); - expect(result).toEqual(resultFields); - }); it('#getDataFromSourceHits', () => { const _source: EventSource = { '@timestamp': '2021-02-24T00:41:06.527Z', @@ -498,4 +189,8 @@ describe('Events Details Helpers', () => { }, ]); }); + it('#getDataSafety', async () => { + const result = await getDataSafety(getDataFromFieldsHits, fields); + expect(result).toEqual(resultFields); + }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts index a0ec0b77c6422c..2fc729729e4351 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/helpers.ts @@ -7,8 +7,12 @@ import { get, isEmpty, isNumber, isObject, isString } from 'lodash/fp'; -import { EventSource, TimelineEventsDetailsItem } from '../../../../../../common/search_strategy'; -import { toObjectArrayOfStrings } from '../../../../helpers/to_array'; +import { + EventHit, + EventSource, + TimelineEventsDetailsItem, +} from '../../../../../../common/search_strategy'; +import { toObjectArrayOfStrings, toStringArray } from '../../../../helpers/to_array'; export const baseCategoryFields = ['@timestamp', 'labels', 'message', 'tags']; @@ -24,15 +28,15 @@ export const formatGeoLocation = (item: unknown[]) => { const itemGeo = item.length > 0 ? (item[0] as { coordinates: number[] }) : null; if (itemGeo != null && !isEmpty(itemGeo.coordinates)) { try { - return toObjectArrayOfStrings({ - long: itemGeo.coordinates[0], + return toStringArray({ + lon: itemGeo.coordinates[0], lat: itemGeo.coordinates[1], - }).map(({ str }) => str); + }); } catch { - return toObjectArrayOfStrings(item).map(({ str }) => str); + return toStringArray(item); } } - return toObjectArrayOfStrings(item).map(({ str }) => str); + return toStringArray(item); }; export const isGeoField = (field: string) => @@ -73,14 +77,27 @@ export const getDataFromSourceHits = ( }, []); export const getDataFromFieldsHits = ( - fields: Record, + fields: EventHit['fields'], prependField?: string, prependFieldCategory?: string ): TimelineEventsDetailsItem[] => Object.keys(fields).reduce((accumulator, field) => { const item: unknown[] = fields[field]; + const fieldCategory = prependFieldCategory != null ? prependFieldCategory : getFieldCategory(field); + if (isGeoField(field)) { + return [ + ...accumulator, + { + category: fieldCategory, + field, + values: formatGeoLocation(item), + originalValue: formatGeoLocation(item), + isObjectArray: true, // important for UI + }, + ]; + } const objArrStr = toObjectArrayOfStrings(item); const strArr = objArrStr.map(({ str }) => str); const isObjectArray = objArrStr.some((o) => o.isObjectArray); @@ -93,10 +110,10 @@ export const getDataFromFieldsHits = ( { category: fieldCategory, field: dotField, - values: isGeoField(field) ? formatGeoLocation(item) : strArr, + values: strArr, originalValue: strArr, isObjectArray, - } as TimelineEventsDetailsItem, + }, ]; } @@ -108,10 +125,13 @@ export const getDataFromFieldsHits = ( : getDataFromFieldsHits(item, prependField, fieldCategory); // combine duplicate fields - const flat = [...accumulator, ...nestedFields].reduce( + const flat: Record = [ + ...accumulator, + ...nestedFields, + ].reduce( (acc, f) => ({ ...acc, - // acc is hashmap to determine if we already have the field or not without an array iteration + // acc/flat is hashmap to determine if we already have the field or not without an array iteration // its converted back to array in return with Object.values ...(acc[f.field] != null ? { @@ -133,12 +153,5 @@ export const getDataFromFieldsHits = ( return Object.values(flat); }, []); -export const getDataFromFieldsHitsSafety = ( - fields: Record -): Promise => { - return new Promise((resolve) => { - setTimeout(() => { - resolve(getDataFromFieldsHits(fields)); - }); - }); -}; +export const getDataSafety = (fn: (args: A) => T, args: A): Promise => + new Promise((resolve) => setTimeout(() => resolve(fn(args)))); diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts index cb721de613f03a..7794de7f0f4119 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/details/index.ts @@ -13,11 +13,13 @@ import { TimelineEventsQueries, TimelineEventsDetailsStrategyResponse, TimelineEventsDetailsRequestOptions, + TimelineEventsDetailsItem, + EventSource, } from '../../../../../../common/search_strategy'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionTimelineFactory } from '../../types'; import { buildTimelineDetailsQuery } from './query.events_details.dsl'; -import { getDataFromFieldsHitsSafety, getDataFromSourceHits } from './helpers'; +import { getDataFromFieldsHits, getDataFromSourceHits, getDataSafety } from './helpers'; export const timelineEventsDetails: SecuritySolutionTimelineFactory = { buildDsl: (options: TimelineEventsDetailsRequestOptions) => { @@ -40,9 +42,14 @@ export const timelineEventsDetails: SecuritySolutionTimelineFactory( + getDataFromSourceHits, + _source + ); + const fieldsData = await getDataSafety( + getDataFromFieldsHits, + merge(fields, hitsData) + ); const data = unionBy('field', fieldsData, sourceData); return { diff --git a/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts new file mode 100644 index 00000000000000..13b7fe70512460 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/timeline/factory/events/mocks.ts @@ -0,0 +1,329 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const eventHit = { + _index: 'auditbeat-7.8.0-2020.11.05-000003', + _id: 'tkCt1nUBaEgqnrVSZ8R_', + _score: 0, + _type: '', + fields: { + 'event.category': ['process'], + 'process.ppid': [3977], + 'user.name': ['jenkins'], + 'process.args': ['go', 'vet', './...'], + message: ['Process go (PID: 4313) by user jenkins STARTED'], + 'process.pid': [4313], + 'process.working_directory': [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + 'process.entity_id': ['Z59cIkAAIw8ZoK0H'], + 'host.ip': ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + 'process.name': ['go'], + 'event.action': ['process_started'], + 'agent.type': ['auditbeat'], + '@timestamp': ['2020-11-17T14:48:08.922Z'], + 'event.module': ['system'], + 'event.type': ['start'], + 'host.name': ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + 'process.hash.sha1': ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + 'host.os.family': ['debian'], + 'event.kind': ['event'], + 'host.id': ['e59991e835905c65ed3e455b33e13bd6'], + 'event.dataset': ['process'], + 'process.executable': [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + 'source.geo.location': [{ coordinates: [118.7778, 32.0617], type: 'Point' }], + 'threat.indicator': [ + { + 'matched.field': ['matched_field'], + first_seen: ['2021-02-22T17:29:25.195Z'], + provider: ['yourself'], + type: ['custom'], + 'matched.atomic': ['matched_atomic'], + lazer: [ + { + 'great.field': ['grrrrr'], + }, + { + 'great.field': ['grrrrr_2'], + }, + ], + }, + { + 'matched.field': ['matched_field_2'], + first_seen: ['2021-02-22T17:29:25.195Z'], + provider: ['other_you'], + type: ['custom'], + 'matched.atomic': ['matched_atomic_2'], + lazer: [ + { + 'great.field': [ + { + wowoe: [ + { + fooooo: ['grrrrr'], + }, + ], + astring: 'cool', + aNumber: 1, + anObject: { + neat: true, + }, + }, + ], + }, + ], + }, + ], + }, + _source: {}, + sort: ['1605624488922', 'beats-ci-immutable-ubuntu-1804-1605624279743236239'], + aggregations: {}, +}; + +export const eventDetailsFormattedFields = [ + { + category: 'event', + field: 'event.category', + isObjectArray: false, + originalValue: ['process'], + values: ['process'], + }, + { + category: 'process', + field: 'process.ppid', + isObjectArray: false, + originalValue: ['3977'], + values: ['3977'], + }, + { + category: 'user', + field: 'user.name', + isObjectArray: false, + originalValue: ['jenkins'], + values: ['jenkins'], + }, + { + category: 'process', + field: 'process.args', + isObjectArray: false, + originalValue: ['go', 'vet', './...'], + values: ['go', 'vet', './...'], + }, + { + category: 'base', + field: 'message', + isObjectArray: false, + originalValue: ['Process go (PID: 4313) by user jenkins STARTED'], + values: ['Process go (PID: 4313) by user jenkins STARTED'], + }, + { + category: 'process', + field: 'process.pid', + isObjectArray: false, + originalValue: ['4313'], + values: ['4313'], + }, + { + category: 'process', + field: 'process.working_directory', + isObjectArray: false, + originalValue: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + values: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/src/github.com/elastic/beats/libbeat', + ], + }, + { + category: 'process', + field: 'process.entity_id', + isObjectArray: false, + originalValue: ['Z59cIkAAIw8ZoK0H'], + values: ['Z59cIkAAIw8ZoK0H'], + }, + { + category: 'host', + field: 'host.ip', + isObjectArray: false, + originalValue: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + values: ['10.224.1.237', 'fe80::4001:aff:fee0:1ed', '172.17.0.1'], + }, + { + category: 'process', + field: 'process.name', + isObjectArray: false, + originalValue: ['go'], + values: ['go'], + }, + { + category: 'event', + field: 'event.action', + isObjectArray: false, + originalValue: ['process_started'], + values: ['process_started'], + }, + { + category: 'agent', + field: 'agent.type', + isObjectArray: false, + originalValue: ['auditbeat'], + values: ['auditbeat'], + }, + { + category: 'base', + field: '@timestamp', + isObjectArray: false, + originalValue: ['2020-11-17T14:48:08.922Z'], + values: ['2020-11-17T14:48:08.922Z'], + }, + { + category: 'event', + field: 'event.module', + isObjectArray: false, + originalValue: ['system'], + values: ['system'], + }, + { + category: 'event', + field: 'event.type', + isObjectArray: false, + originalValue: ['start'], + values: ['start'], + }, + { + category: 'host', + field: 'host.name', + isObjectArray: false, + originalValue: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + values: ['beats-ci-immutable-ubuntu-1804-1605624279743236239'], + }, + { + category: 'process', + field: 'process.hash.sha1', + isObjectArray: false, + originalValue: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + values: ['1eac22336a41e0660fb302add9d97daa2bcc7040'], + }, + { + category: 'host', + field: 'host.os.family', + isObjectArray: false, + originalValue: ['debian'], + values: ['debian'], + }, + { + category: 'event', + field: 'event.kind', + isObjectArray: false, + originalValue: ['event'], + values: ['event'], + }, + { + category: 'host', + field: 'host.id', + isObjectArray: false, + originalValue: ['e59991e835905c65ed3e455b33e13bd6'], + values: ['e59991e835905c65ed3e455b33e13bd6'], + }, + { + category: 'event', + field: 'event.dataset', + isObjectArray: false, + originalValue: ['process'], + values: ['process'], + }, + { + category: 'process', + field: 'process.executable', + isObjectArray: false, + originalValue: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + values: [ + '/var/lib/jenkins/workspace/Beats_beats_PR-22624/.gvm/versions/go1.14.7.linux.amd64/bin/go', + ], + }, + { + category: 'source', + field: 'source.geo.location', + isObjectArray: true, + originalValue: [`{"lon":118.7778,"lat":32.0617}`], + values: [`{"lon":118.7778,"lat":32.0617}`], + }, + { + category: 'threat', + field: 'threat.indicator.matched.field', + values: ['matched_field', 'matched_field_2'], + originalValue: ['matched_field', 'matched_field_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.first_seen', + values: ['2021-02-22T17:29:25.195Z'], + originalValue: ['2021-02-22T17:29:25.195Z'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.provider', + values: ['yourself', 'other_you'], + originalValue: ['yourself', 'other_you'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.type', + values: ['custom'], + originalValue: ['custom'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.matched.atomic', + values: ['matched_atomic', 'matched_atomic_2'], + originalValue: ['matched_atomic', 'matched_atomic_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field', + values: ['grrrrr', 'grrrrr_2'], + originalValue: ['grrrrr', 'grrrrr_2'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.wowoe.fooooo', + values: ['grrrrr'], + originalValue: ['grrrrr'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.astring', + values: ['cool'], + originalValue: ['cool'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.aNumber', + values: ['1'], + originalValue: ['1'], + isObjectArray: false, + }, + { + category: 'threat', + field: 'threat.indicator.lazer.great.field.neat', + values: ['true'], + originalValue: ['true'], + isObjectArray: false, + }, +];