From c51116a0dff05901665a686de21cbb841ab0a2a2 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Tue, 17 Mar 2020 17:40:21 -0400 Subject: [PATCH] [Alerting] add functional tests for index threshold alertType resolves https://github.com/elastic/kibana/issues/58902 --- .../alert_types/index_threshold/alert_type.ts | 2 + .../common/lib/es_test_index_tool.ts | 23 +- .../index_threshold/alert.ts | 398 ++++++++++++++++++ .../index_threshold/create_test_data.ts | 48 +-- .../index_threshold/index.ts | 1 + .../time_series_query_endpoint.ts | 24 +- 6 files changed, 447 insertions(+), 49 deletions(-) create mode 100644 x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/alert.ts diff --git a/x-pack/plugins/alerting_builtins/server/alert_types/index_threshold/alert_type.ts b/x-pack/plugins/alerting_builtins/server/alert_types/index_threshold/alert_type.ts index b79321a8803fad..6d27f8a99dd4b5 100644 --- a/x-pack/plugins/alerting_builtins/server/alert_types/index_threshold/alert_type.ts +++ b/x-pack/plugins/alerting_builtins/server/alert_types/index_threshold/alert_type.ts @@ -113,6 +113,7 @@ export function getAlertType(service: Service): AlertType { timeWindowUnit: params.timeWindowUnit, interval: undefined, }; + // console.log(`index_threshold: query: ${JSON.stringify(queryParams, null, 4)}`); const result = await service.indexThreshold.timeSeriesQuery({ logger, callCluster, @@ -121,6 +122,7 @@ export function getAlertType(service: Service): AlertType { logger.debug(`alert ${ID}:${alertId} "${name}" query result: ${JSON.stringify(result)}`); const groupResults = result.results || []; + // console.log(`index_threshold: response: ${JSON.stringify(groupResults, null, 4)}`); for (const groupResult of groupResults) { const instanceId = groupResult.group; const value = groupResult.metrics[0][1]; diff --git a/x-pack/test/alerting_api_integration/common/lib/es_test_index_tool.ts b/x-pack/test/alerting_api_integration/common/lib/es_test_index_tool.ts index ccd7748d9e899d..999a8686e0ee7a 100644 --- a/x-pack/test/alerting_api_integration/common/lib/es_test_index_tool.ts +++ b/x-pack/test/alerting_api_integration/common/lib/es_test_index_tool.ts @@ -4,20 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -export const ES_TEST_INDEX_NAME = '.kibaka-alerting-test-data'; +export const ES_TEST_INDEX_NAME = '.kibana-alerting-test-data'; export class ESTestIndexTool { - private readonly es: any; - private readonly retry: any; - - constructor(es: any, retry: any) { - this.es = es; - this.retry = retry; - } + constructor( + private readonly es: any, + private readonly retry: any, + private readonly index: string = ES_TEST_INDEX_NAME + ) {} async setup() { return await this.es.indices.create({ - index: ES_TEST_INDEX_NAME, + index: this.index, body: { mappings: { properties: { @@ -56,12 +54,13 @@ export class ESTestIndexTool { } async destroy() { - return await this.es.indices.delete({ index: ES_TEST_INDEX_NAME, ignore: [404] }); + return await this.es.indices.delete({ index: this.index, ignore: [404] }); } async search(source: string, reference: string) { return await this.es.search({ - index: ES_TEST_INDEX_NAME, + index: this.index, + size: 1000, body: { query: { bool: { @@ -86,7 +85,7 @@ export class ESTestIndexTool { async waitForDocs(source: string, reference: string, numDocs: number = 1) { return await this.retry.try(async () => { const searchResult = await this.search(source, reference); - if (searchResult.hits.total.value !== numDocs) { + if (searchResult.hits.total.value < numDocs) { throw new Error(`Expected ${numDocs} but received ${searchResult.hits.total.value}.`); } return searchResult.hits.hits; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/alert.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/alert.ts new file mode 100644 index 00000000000000..13f3a4971183c6 --- /dev/null +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/alert.ts @@ -0,0 +1,398 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { Spaces } from '../../../../scenarios'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { + ESTestIndexTool, + ES_TEST_INDEX_NAME, + getUrlPrefix, + ObjectRemover, +} from '../../../../../common/lib'; +import { createEsDocuments } from './create_test_data'; + +const ALERT_TYPE_ID = '.index-threshold'; +const ACTION_TYPE_ID = '.index'; +const ES_TEST_INDEX_SOURCE = 'builtin-alert:index-threshold'; +const ES_TEST_INDEX_REFERENCE = '-na-'; +const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`; + +const ALERT_INTERVALS_TO_WRITE = 5; +const ALERT_INTERVAL_SECONDS = 3; +const ALERT_INTERVAL_MILLIS = ALERT_INTERVAL_SECONDS * 1000; + +// eslint-disable-next-line import/no-default-export +export default function alertTests({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const retry = getService('retry'); + const es = getService('legacyEs'); + const esTestIndexTool = new ESTestIndexTool(es, retry); + const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME); + + describe('alert', async () => { + let endDate: string; + let actionId: string; + const objectRemover = new ObjectRemover(supertest); + + beforeEach(async () => { + await esTestIndexTool.destroy(); + await esTestIndexTool.setup(); + + await esTestIndexToolOutput.destroy(); + await esTestIndexToolOutput.setup(); + + actionId = await createAction(supertest, objectRemover); + + // write documents in the future, figure out the end date + const endDateMillis = Date.now() + (ALERT_INTERVALS_TO_WRITE - 1) * ALERT_INTERVAL_MILLIS; + endDate = new Date(endDateMillis).toISOString(); + + // write documents from now to the future end date in 3 groups + createEsDocumentsInGroups(3); + }); + + afterEach(async () => { + await objectRemover.removeAll(); + await esTestIndexTool.destroy(); + await esTestIndexToolOutput.destroy(); + }); + + // The tests below create two alerts, one that will fire, one that will + // never fire; the tests ensure the ones that should fire, do fire, and + // those that shouldn't fire, do not fire. + it('runs correctly: count all < >', async () => { + await createAlert({ + name: 'never fire', + aggType: 'count', + groupBy: 'all', + thresholdComparator: '<', + threshold: [0], + }); + + await createAlert({ + name: 'always fire', + aggType: 'count', + groupBy: 'all', + thresholdComparator: '>', + threshold: [-1], + }); + + const docs = await waitForDocs(2); + for (const doc of docs) { + const { group } = doc._source; + const { name, value, title, message } = doc._source.params; + + expect(name).to.be('always fire'); + expect(group).to.be('all documents'); + + // we'll check title and message in this test, but not subsequent ones + expect(title).to.be('alert always fire group all documents exceeded threshold'); + + const expectedPrefix = `alert always fire group all documents value ${value} exceeded threshold count > -1 over`; + const messagePrefix = message.substr(0, expectedPrefix.length); + expect(messagePrefix).to.be(expectedPrefix); + } + }); + + it('runs correctly: count grouped <= =>', async () => { + // create some more documents in the first group + createEsDocumentsInGroups(1); + + await createAlert({ + name: 'never fire', + aggType: 'count', + groupBy: 'top', + termField: 'group', + termSize: 2, + thresholdComparator: '<=', + threshold: [-1], + }); + + await createAlert({ + name: 'always fire', + aggType: 'count', + groupBy: 'top', + termField: 'group', + termSize: 2, // two actions will fire each interval + thresholdComparator: '>=', + threshold: [0], + }); + + const docs = await waitForDocs(4); + let inGroup0 = 0; + + for (const doc of docs) { + const { group } = doc._source; + const { name } = doc._source.params; + + expect(name).to.be('always fire'); + if (group === 'group-0') inGroup0++; + } + + // there should be 2 docs in group-0, rando split between others + expect(inGroup0).to.be(2); + }); + + it('runs correctly: sum all between', async () => { + // create some more documents in the first group + createEsDocumentsInGroups(1); + + await createAlert({ + name: 'never fire', + aggType: 'sum', + aggField: 'testedValue', + groupBy: 'all', + thresholdComparator: 'between', + threshold: [-2, -1], + }); + + await createAlert({ + name: 'always fire', + aggType: 'sum', + aggField: 'testedValue', + groupBy: 'all', + thresholdComparator: 'between', + threshold: [0, 1000000], + }); + + const docs = await waitForDocs(2); + for (const doc of docs) { + const { name } = doc._source.params; + + expect(name).to.be('always fire'); + } + }); + + it('runs correctly: avg all', async () => { + // create some more documents in the first group + createEsDocumentsInGroups(1); + + await createAlert({ + name: 'never fire', + aggType: 'avg', + aggField: 'testedValue', + groupBy: 'all', + thresholdComparator: '<', + threshold: [0], + }); + + await createAlert({ + name: 'always fire', + aggType: 'avg', + aggField: 'testedValue', + groupBy: 'all', + thresholdComparator: '>=', + threshold: [0], + }); + + const docs = await waitForDocs(4); + for (const doc of docs) { + const { name } = doc._source.params; + + expect(name).to.be('always fire'); + } + }); + + it('runs correctly: max grouped', async () => { + // create some more documents in the first group + createEsDocumentsInGroups(1); + + await createAlert({ + name: 'never fire', + aggType: 'max', + aggField: 'testedValue', + groupBy: 'top', + termField: 'group', + termSize: 2, + thresholdComparator: '<', + threshold: [0], + }); + + await createAlert({ + name: 'always fire', + aggType: 'max', + aggField: 'testedValue', + groupBy: 'top', + termField: 'group', + termSize: 2, // two actions will fire each interval + thresholdComparator: '>=', + threshold: [0], + }); + + const docs = await waitForDocs(4); + let inGroup2 = 0; + + for (const doc of docs) { + const { group } = doc._source; + const { name } = doc._source.params; + + expect(name).to.be('always fire'); + if (group === 'group-2') inGroup2++; + } + + // there should be 2 docs in group-2, rando split between others + expect(inGroup2).to.be(2); + }); + + it('runs correctly: min grouped', async () => { + // create some more documents in the first group + createEsDocumentsInGroups(1); + + await createAlert({ + name: 'never fire', + aggType: 'min', + aggField: 'testedValue', + groupBy: 'top', + termField: 'group', + termSize: 2, + thresholdComparator: '<', + threshold: [0], + }); + + await createAlert({ + name: 'always fire', + aggType: 'min', + aggField: 'testedValue', + groupBy: 'top', + termField: 'group', + termSize: 2, // two actions will fire each interval + thresholdComparator: '>=', + threshold: [0], + }); + + const docs = await waitForDocs(4); + let inGroup0 = 0; + + for (const doc of docs) { + const { group } = doc._source; + const { name } = doc._source.params; + + expect(name).to.be('always fire'); + if (group === 'group-0') inGroup0++; + } + + // there should be 2 docs in group-0, rando split between others + expect(inGroup0).to.be(2); + }); + + async function createEsDocumentsInGroups(groups: number) { + await createEsDocuments( + es, + esTestIndexTool, + endDate, + ALERT_INTERVALS_TO_WRITE, + ALERT_INTERVAL_MILLIS, + groups + ); + } + + async function waitForDocs(count: number): Promise { + return await esTestIndexToolOutput.waitForDocs( + ES_TEST_INDEX_SOURCE, + ES_TEST_INDEX_REFERENCE, + count + ); + } + + interface CreateAlertParams { + name: string; + aggType: string; + aggField?: string; + groupBy: 'all' | 'top'; + termField?: string; + termSize?: number; + thresholdComparator: string; + threshold: number[]; + } + + async function createAlert(params: CreateAlertParams): Promise { + const action = { + id: actionId, + group: 'threshold met', + params: { + documents: [ + { + source: ES_TEST_INDEX_SOURCE, + reference: ES_TEST_INDEX_REFERENCE, + params: { + name: '{{{alertName}}}', + value: '{{{context.value}}}', + title: '{{{context.title}}}', + message: '{{{context.message}}}', + }, + date: '{{{context.date}}}', + // TODO: I wanted to write the alert value here, but how? + // We only mustache interpolate string values ... + // testedValue: '{{{context.value}}}', + group: '{{{context.group}}}', + }, + ], + }, + }; + + const { statusCode, body: createdAlert } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alert`) + .set('kbn-xsrf', 'foo') + .send({ + name: params.name, + consumer: 'function test', + enabled: true, + alertTypeId: ALERT_TYPE_ID, + schedule: { interval: `${ALERT_INTERVAL_SECONDS}s` }, + actions: [action], + params: { + index: ES_TEST_INDEX_NAME, + timeField: 'date', + aggType: params.aggType, + aggField: params.aggField, + groupBy: params.groupBy, + termField: params.termField, + termSize: params.termSize, + timeWindowSize: ALERT_INTERVAL_SECONDS * 5, + timeWindowUnit: 's', + thresholdComparator: params.thresholdComparator, + threshold: params.threshold, + }, + }); + + // will print the error body, if an error occurred + // if (statusCode !== 200) console.log(createdAlert); + + expect(statusCode).to.be(200); + + const alertId = createdAlert.id; + objectRemover.add(Spaces.space1.id, alertId, 'alert'); + + return alertId; + } + }); +} + +async function createAction(supertest: any, objectRemover: ObjectRemover): Promise { + const { statusCode, body: createdAction } = await supertest + .post(`${getUrlPrefix(Spaces.space1.id)}/api/action`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'index action for index threshold FT', + actionTypeId: ACTION_TYPE_ID, + config: { + index: ES_TEST_OUTPUT_INDEX_NAME, + }, + secrets: {}, + }); + + // will print the error body, if an error occurred + // if (statusCode !== 200) console.log(createdAction); + + expect(statusCode).to.be(200); + + const actionId = createdAction.id; + objectRemover.add(Spaces.space1.id, actionId, 'action'); + + return actionId; +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/create_test_data.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/create_test_data.ts index 523c348e260497..21f73ac9b98330 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/create_test_data.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/create_test_data.ts @@ -8,53 +8,50 @@ import { times } from 'lodash'; import { v4 as uuid } from 'uuid'; import { ESTestIndexTool, ES_TEST_INDEX_NAME } from '../../../../../common/lib'; -// date to start writing data -export const START_DATE = '2020-01-01T00:00:00Z'; +// default end date +export const END_DATE = '2020-01-01T00:00:00Z'; -const DOCUMENT_SOURCE = 'queryDataEndpointTests'; +export const DOCUMENT_SOURCE = 'queryDataEndpointTests'; +export const DOCUMENT_REFERENCE = '-na-'; // Create a set of es documents to run the queries against. -// Will create 2 documents for each interval. +// Will create `groups` documents for each interval. // The difference between the dates of the docs will be intervalMillis. // The date of the last documents will be startDate - intervalMillis / 2. -// So there will be 2 documents written in the middle of each interval range. -// The data value written to each doc is a power of 2, with 2^0 as the value -// of the last documents, the values increasing for older documents. The -// second document for each time value will be power of 2 + 1 +// So the documents will be written in the middle of each interval range. +// The data value written to each doc is a power of 2 + the group index, with +// 2^0 as the value of the last documents, the values increasing for older +// documents. export async function createEsDocuments( es: any, esTestIndexTool: ESTestIndexTool, - startDate: string = START_DATE, + endDate: string = END_DATE, intervals: number = 1, - intervalMillis: number = 1000 + intervalMillis: number = 1000, + groups: number = 2 ) { - const totalDocuments = intervals * 2; - const startDateMillis = Date.parse(startDate) - intervalMillis / 2; + const endDateMillis = Date.parse(endDate) - intervalMillis / 2; times(intervals, interval => { - const date = startDateMillis - interval * intervalMillis; + const date = endDateMillis - interval * intervalMillis; - // base value for each window is 2^window + // base value for each window is 2^interval const testedValue = 2 ** interval; // don't need await on these, wait at the end of the function - createEsDocument(es, '-na-', date, testedValue, 'groupA'); - createEsDocument(es, '-na-', date, testedValue + 1, 'groupB'); + times(groups, group => { + createEsDocument(es, date, testedValue + group, `group-${group}`); + }); }); - await esTestIndexTool.waitForDocs(DOCUMENT_SOURCE, '-na-', totalDocuments); + const totalDocuments = intervals * groups; + await esTestIndexTool.waitForDocs(DOCUMENT_SOURCE, DOCUMENT_REFERENCE, totalDocuments); } -async function createEsDocument( - es: any, - reference: string, - epochMillis: number, - testedValue: number, - group: string -) { +async function createEsDocument(es: any, epochMillis: number, testedValue: number, group: string) { const document = { source: DOCUMENT_SOURCE, - reference, + reference: DOCUMENT_REFERENCE, date: new Date(epochMillis).toISOString(), testedValue, group, @@ -65,6 +62,7 @@ async function createEsDocument( index: ES_TEST_INDEX_NAME, body: document, }); + // console.log(`writing document to ${ES_TEST_INDEX_NAME}:`, JSON.stringify(document, null, 4)); if (response.result !== 'created') { throw new Error(`document not created: ${JSON.stringify(response)}`); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/index.ts index 9158954f233643..507548f94aaf3e 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/index.ts @@ -12,5 +12,6 @@ export default function alertingTests({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./time_series_query_endpoint')); loadTestFile(require.resolve('./fields_endpoint')); loadTestFile(require.resolve('./indices_endpoint')); + loadTestFile(require.resolve('./alert')); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/time_series_query_endpoint.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/time_series_query_endpoint.ts index 1aa1d3d21f00d1..c9b488da5dec5d 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/time_series_query_endpoint.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/builtin_alert_types/index_threshold/time_series_query_endpoint.ts @@ -39,12 +39,12 @@ const START_DATE_MINUS_2INTERVALS = getStartDate(-2 * INTERVAL_MILLIS); are offset from the top of the minute by 30 seconds, the queries always run from the top of the hour. - { "date":"2019-12-31T23:59:30.000Z", "testedValue":1, "group":"groupA" } - { "date":"2019-12-31T23:59:30.000Z", "testedValue":2, "group":"groupB" } - { "date":"2019-12-31T23:58:30.000Z", "testedValue":2, "group":"groupA" } - { "date":"2019-12-31T23:58:30.000Z", "testedValue":3, "group":"groupB" } - { "date":"2019-12-31T23:57:30.000Z", "testedValue":4, "group":"groupA" } - { "date":"2019-12-31T23:57:30.000Z", "testedValue":5, "group":"groupB" } + { "date":"2019-12-31T23:59:30.000Z", "testedValue":1, "group":"group-0" } + { "date":"2019-12-31T23:59:30.000Z", "testedValue":2, "group":"group-1" } + { "date":"2019-12-31T23:58:30.000Z", "testedValue":2, "group":"group-0" } + { "date":"2019-12-31T23:58:30.000Z", "testedValue":3, "group":"group-1" } + { "date":"2019-12-31T23:57:30.000Z", "testedValue":4, "group":"group-0" } + { "date":"2019-12-31T23:57:30.000Z", "testedValue":5, "group":"group-1" } */ // eslint-disable-next-line import/no-default-export @@ -162,7 +162,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider const expected = { results: [ { - group: 'groupA', + group: 'group-0', metrics: [ [START_DATE_MINUS_2INTERVALS, 1], [START_DATE_MINUS_1INTERVALS, 2], @@ -170,7 +170,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider ], }, { - group: 'groupB', + group: 'group-1', metrics: [ [START_DATE_MINUS_2INTERVALS, 1], [START_DATE_MINUS_1INTERVALS, 2], @@ -197,7 +197,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider const expected = { results: [ { - group: 'groupB', + group: 'group-1', metrics: [ [START_DATE_MINUS_2INTERVALS, 5 / 1], [START_DATE_MINUS_1INTERVALS, (5 + 3) / 2], @@ -205,7 +205,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider ], }, { - group: 'groupA', + group: 'group-0', metrics: [ [START_DATE_MINUS_2INTERVALS, 4 / 1], [START_DATE_MINUS_1INTERVALS, (4 + 2) / 2], @@ -230,7 +230,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider }); const result = await runQueryExpect(query, 200); expect(result.results.length).to.be(1); - expect(result.results[0].group).to.be('groupB'); + expect(result.results[0].group).to.be('group-1'); }); it('should return correct sorted group for min', async () => { @@ -245,7 +245,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider }); const result = await runQueryExpect(query, 200); expect(result.results.length).to.be(1); - expect(result.results[0].group).to.be('groupA'); + expect(result.results[0].group).to.be('group-0'); }); it('should return an error when passed invalid input', async () => {