Skip to content

Commit

Permalink
[Security Solution] Use safe type in resolver backend (#76969)
Browse files Browse the repository at this point in the history 
* Moving generator to safe type version

* Finished generator and alert

* Gzipping again

* Finishing type conversions for backend

* Trying to cast front end tests back to unsafe type for now

* Working reducer tests

* Adding more comments and fixing alert type

* Restoring resolver test data

* Updating snapshot with timestamp info

* Removing todo and fixing test

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
  • Loading branch information
@jonathan-buttner @elasticmachine
jonathan-buttner and elasticmachine committed Sep 10, 2020
1 parent eacd602 commit e7b02d0
Show file tree
Hide file tree
Showing 38 changed files with 794 additions and 582 deletions.
142 changes: 81 additions & 61 deletions x-pack/plugins/security_solution/common/endpoint/generate_data.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ import {
ECSCategory,
ANCESTRY_LIMIT,
} from './generate_data';
import { firstNonNullValue, values } from './models/ecs_safety_helpers';
import {
entityIDSafeVersion,
parentEntityIDSafeVersion,
timestampSafeVersion,
} from './models/event';

interface Node {
events: Event[];
Expand All @@ -30,7 +36,7 @@ describe('data generator', () => {
const event1 = generator.generateEvent();
const event2 = generator.generateEvent();

expect(event2.event.sequence).toBe(event1.event.sequence + 1);
expect(event2.event?.sequence).toBe((firstNonNullValue(event1.event?.sequence) ?? 0) + 1);
});

it('creates the same documents with same random seed', () => {
Expand Down Expand Up @@ -76,37 +82,37 @@ describe('data generator', () => {
const timestamp = new Date().getTime();
const alert = generator.generateAlert(timestamp);
expect(alert['@timestamp']).toEqual(timestamp);
expect(alert.event.action).not.toBeNull();
expect(alert.event?.action).not.toBeNull();
expect(alert.Endpoint).not.toBeNull();
expect(alert.agent).not.toBeNull();
expect(alert.host).not.toBeNull();
expect(alert.process.entity_id).not.toBeNull();
expect(alert.process?.entity_id).not.toBeNull();
});

it('creates process event documents', () => {
const timestamp = new Date().getTime();
const processEvent = generator.generateEvent({ timestamp });
expect(processEvent['@timestamp']).toEqual(timestamp);
expect(processEvent.event.category).toEqual(['process']);
expect(processEvent.event.kind).toEqual('event');
expect(processEvent.event.type).toEqual(['start']);
expect(processEvent.event?.category).toEqual(['process']);
expect(processEvent.event?.kind).toEqual('event');
expect(processEvent.event?.type).toEqual(['start']);
expect(processEvent.agent).not.toBeNull();
expect(processEvent.host).not.toBeNull();
expect(processEvent.process.entity_id).not.toBeNull();
expect(processEvent.process.name).not.toBeNull();
expect(processEvent.process?.entity_id).not.toBeNull();
expect(processEvent.process?.name).not.toBeNull();
});

it('creates other event documents', () => {
const timestamp = new Date().getTime();
const processEvent = generator.generateEvent({ timestamp, eventCategory: 'dns' });
expect(processEvent['@timestamp']).toEqual(timestamp);
expect(processEvent.event.category).toEqual('dns');
expect(processEvent.event.kind).toEqual('event');
expect(processEvent.event.type).toEqual(['start']);
expect(processEvent.event?.category).toEqual('dns');
expect(processEvent.event?.kind).toEqual('event');
expect(processEvent.event?.type).toEqual(['start']);
expect(processEvent.agent).not.toBeNull();
expect(processEvent.host).not.toBeNull();
expect(processEvent.process.entity_id).not.toBeNull();
expect(processEvent.process.name).not.toBeNull();
expect(processEvent.process?.entity_id).not.toBeNull();
expect(processEvent.process?.name).not.toBeNull();
});

describe('creates events with an empty ancestry array', () => {
Expand All @@ -128,7 +134,7 @@ describe('data generator', () => {

it('creates all events with an empty ancestry array', () => {
for (const event of tree.allEvents) {
expect(event.process.Ext!.ancestry!.length).toEqual(0);
expect(event.process?.Ext?.ancestry?.length).toEqual(0);
}
});
});
Expand Down Expand Up @@ -194,36 +200,38 @@ describe('data generator', () => {
const inRelated = node.relatedEvents.includes(event);
const inRelatedAlerts = node.relatedAlerts.includes(event);

return (inRelated || inRelatedAlerts || inLifecycle) && event.process.entity_id === node.id;
return (inRelated || inRelatedAlerts || inLifecycle) && event.process?.entity_id === node.id;
};

const verifyAncestry = (event: Event, genTree: Tree) => {
if (event.process.Ext!.ancestry!.length > 0) {
expect(event.process.parent?.entity_id).toBe(event.process.Ext!.ancestry![0]);
const ancestry = values(event.process?.Ext?.ancestry);
if (ancestry.length > 0) {
expect(event.process?.parent?.entity_id).toBe(ancestry[0]);
}
for (let i = 0; i < event.process.Ext!.ancestry!.length; i++) {
const ancestor = event.process.Ext!.ancestry![i];
for (let i = 0; i < ancestry.length; i++) {
const ancestor = ancestry[i];
const parent = genTree.children.get(ancestor) || genTree.ancestry.get(ancestor);
expect(ancestor).toBe(parent?.lifecycle[0].process.entity_id);
expect(ancestor).toBe(parent?.lifecycle[0].process?.entity_id);

// the next ancestor should be the grandparent
if (i + 1 < event.process.Ext!.ancestry!.length) {
const grandparent = event.process.Ext!.ancestry![i + 1];
expect(grandparent).toBe(parent?.lifecycle[0].process.parent?.entity_id);
if (i + 1 < ancestry.length) {
const grandparent = ancestry[i + 1];
expect(grandparent).toBe(parent?.lifecycle[0].process?.parent?.entity_id);
}
}
};

it('creates related events in ascending order', () => {
// the order should not change since it should already be in ascending order
const relatedEventsAsc = _.cloneDeep(tree.origin.relatedEvents).sort(
(event1, event2) => event1['@timestamp'] - event2['@timestamp']
(event1, event2) =>
(timestampSafeVersion(event1) ?? 0) - (timestampSafeVersion(event2) ?? 0)
);
expect(tree.origin.relatedEvents).toStrictEqual(relatedEventsAsc);
});

it('has ancestry array defined', () => {
expect(tree.origin.lifecycle[0].process.Ext!.ancestry!.length).toBe(ANCESTRY_LIMIT);
expect(values(tree.origin.lifecycle[0].process?.Ext?.ancestry).length).toBe(ANCESTRY_LIMIT);
for (const event of tree.allEvents) {
verifyAncestry(event, tree);
}
Expand Down Expand Up @@ -252,12 +260,9 @@ describe('data generator', () => {

const counts: Record<string, number> = {};
for (const event of node.relatedEvents) {
if (Array.isArray(event.event.category)) {
for (const cat of event.event.category) {
counts[cat] = counts[cat] + 1 || 1;
}
} else {
counts[event.event.category] = counts[event.event.category] + 1 || 1;
const categories = values(event.event?.category);
for (const cat of categories) {
counts[cat] = counts[cat] + 1 || 1;
}
}
expect(counts[ECSCategory.Driver]).toEqual(1);
Expand Down Expand Up @@ -316,15 +321,18 @@ describe('data generator', () => {
expect(tree.allEvents.length).toBeGreaterThan(0);

tree.allEvents.forEach((event) => {
const ancestor = tree.ancestry.get(event.process.entity_id);
if (ancestor) {
expect(eventInNode(event, ancestor)).toBeTruthy();
return;
}
const entityID = entityIDSafeVersion(event);
if (entityID) {
const ancestor = tree.ancestry.get(entityID);
if (ancestor) {
expect(eventInNode(event, ancestor)).toBeTruthy();
return;
}

const children = tree.children.get(event.process.entity_id);
if (children) {
expect(eventInNode(event, children)).toBeTruthy();
const children = tree.children.get(entityID);
if (children) {
expect(eventInNode(event, children)).toBeTruthy();
}
}
});
});
Expand All @@ -351,9 +359,8 @@ describe('data generator', () => {
let events: Event[];

const isCategoryProcess = (event: Event) => {
return (
_.isEqual(event.event.category, ['process']) || _.isEqual(event.event.category, 'process')
);
const category = values(event.event?.category);
return _.isEqual(category, ['process']);
};

beforeEach(() => {
Expand All @@ -366,12 +373,16 @@ describe('data generator', () => {

it('with n-1 process events', () => {
for (let i = events.length - 2; i > 0; ) {
const parentEntityIdOfChild = events[i].process.parent?.entity_id;
for (; --i >= -1 && (events[i].event.kind !== 'event' || !isCategoryProcess(events[i])); ) {
const parentEntityIdOfChild = parentEntityIDSafeVersion(events[i]);
for (
;
--i >= -1 && (events[i].event?.kind !== 'event' || !isCategoryProcess(events[i]));

) {
// related event - skip it
}
expect(i).toBeGreaterThanOrEqual(0);
expect(parentEntityIdOfChild).toEqual(events[i].process.entity_id);
expect(parentEntityIdOfChild).toEqual(entityIDSafeVersion(events[i]));
}
});

Expand All @@ -380,37 +391,40 @@ describe('data generator', () => {
for (
;
previousProcessEventIndex >= -1 &&
(events[previousProcessEventIndex].event.kind !== 'event' ||
(events[previousProcessEventIndex].event?.kind !== 'event' ||
!isCategoryProcess(events[previousProcessEventIndex]));
previousProcessEventIndex--
) {
// related event - skip it
}
expect(previousProcessEventIndex).toBeGreaterThanOrEqual(0);
// The alert should be last and have the same entity_id as the previous process event
expect(events[events.length - 1].process.entity_id).toEqual(
events[previousProcessEventIndex].process.entity_id
expect(events[events.length - 1].process?.entity_id).toEqual(
events[previousProcessEventIndex].process?.entity_id
);
expect(events[events.length - 1].process.parent?.entity_id).toEqual(
events[previousProcessEventIndex].process.parent?.entity_id
expect(events[events.length - 1].process?.parent?.entity_id).toEqual(
events[previousProcessEventIndex].process?.parent?.entity_id
);
expect(events[events.length - 1].event.kind).toEqual('alert');
expect(events[events.length - 1].event.category).toEqual('malware');
expect(events[events.length - 1].event?.kind).toEqual('alert');
expect(events[events.length - 1].event?.category).toEqual('malware');
});
});

function buildResolverTree(events: Event[]): Node {
// First pass we gather up all the events by entity_id
const tree: Record<string, Node> = {};
events.forEach((event) => {
if (event.process.entity_id in tree) {
tree[event.process.entity_id].events.push(event);
} else {
tree[event.process.entity_id] = {
events: [event],
children: [],
parent_entity_id: event.process.parent?.entity_id,
};
const entityID = entityIDSafeVersion(event);
if (entityID) {
if (entityID in tree) {
tree[entityID].events.push(event);
} else {
tree[entityID] = {
events: [event],
children: [],
parent_entity_id: parentEntityIDSafeVersion(event),
};
}
}
});
// Second pass add child references to each node
Expand All @@ -419,8 +433,14 @@ describe('data generator', () => {
tree[value.parent_entity_id].children.push(value);
}
}

const entityID = entityIDSafeVersion(events[0]);
if (!entityID) {
throw new Error('entity id was invalid');
}

// The root node must be first in the array or this fails
return tree[events[0].process.entity_id];
return tree[entityID];
}

function countResolverEvents(rootNode: Node, generations: number): number {
Expand Down
Loading

0 comments on commit e7b02d0

Please sign in to comment.