diff --git a/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts new file mode 100644 index 00000000000000..283f5fb8909f64 --- /dev/null +++ b/x-pack/plugins/uptime/server/lib/requests/search/__tests__/refine_potential_matches.test.ts @@ -0,0 +1,112 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { fullyMatchingIds } from '../refine_potential_matches'; +import { MonitorLocCheckGroup } from '..'; + +const mockQueryResult = (opts: { latestSummary: any; latestMatching: any }) => { + return { + aggregations: { + monitor: { + buckets: [ + { + key: 'my-monitor', + location: { + buckets: [ + { + key: 'my-location', + summaries: { + latest: { + hits: { + hits: [ + { + _source: opts.latestSummary, + }, + ], + }, + }, + }, + latest_matching: { + top: { + hits: { + hits: [ + { + _source: opts.latestMatching, + }, + ], + }, + }, + }, + }, + ], + }, + }, + ], + }, + }, + }; +}; + +describe('fully matching IDs', () => { + it('should exclude items whose latest result does not match', () => { + const queryRes = mockQueryResult({ + latestSummary: { + '@timestamp': '2020-06-04T12:39:54.698-0500', + monitor: { + check_group: 'latest-summary-check-group', + }, + summary: { + up: 1, + down: 0, + }, + }, + latestMatching: { + '@timestamp': '2019-06-04T12:39:54.698-0500', + summary: { + up: 1, + down: 0, + }, + }, + }); + const res = fullyMatchingIds(queryRes, undefined); + const expected = new Map(); + expect(res).toEqual(expected); + }); + + it('should include items whose latest result does match', () => { + const queryRes = mockQueryResult({ + latestSummary: { + '@timestamp': '2020-06-04T12:39:54.698-0500', + monitor: { + check_group: 'latest-summary-check-group', + }, + summary: { + up: 1, + down: 0, + }, + }, + latestMatching: { + '@timestamp': '2020-06-04T12:39:54.698-0500', + summary: { + up: 1, + down: 0, + }, + }, + }); + const res = fullyMatchingIds(queryRes, undefined); + const expected = new Map(); + expected.set('my-monitor', [ + { + checkGroup: 'latest-summary-check-group', + location: 'my-location', + monitorId: 'my-monitor', + status: 'up', + summaryTimestamp: new Date('2020-06-04T12:39:54.698-0500'), + }, + ]); + expect(res).toEqual(expected); + }); +}); diff --git a/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts b/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts index 77676ac9a63731..2a5f1f1261cb37 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/fetch_chunk.ts @@ -26,12 +26,12 @@ export const fetchChunk: ChunkFetcher = async ( searchAfter: any, size: number ): Promise => { - const { monitorIds, checkGroups, searchAfter: foundSearchAfter } = await findPotentialMatches( + const { monitorIds, searchAfter: foundSearchAfter } = await findPotentialMatches( queryContext, searchAfter, size ); - const matching = await refinePotentialMatches(queryContext, monitorIds, checkGroups); + const matching = await refinePotentialMatches(queryContext, monitorIds); return { monitorGroups: matching, diff --git a/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts b/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts index a3e7324086073f..ac4ff91230b959 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/find_potential_matches.ts @@ -8,12 +8,8 @@ import { get, set } from 'lodash'; import { CursorDirection } from '../../../../common/runtime_types'; import { QueryContext } from './query_context'; -// This is the first phase of the query. In it, we find the most recent check groups that matched the given query. -// Note that these check groups may not be the most recent groups for the matching monitor ID! We'll filter those /** - * This is the first phase of the query. In it, we find the most recent check groups that matched the given query. - * Note that these check groups may not be the most recent groups for the matching monitor ID. They'll be filtered - * out in the next phase. + * This is the first phase of the query. In it, we find all monitor IDs that have ever matched the given filters. * @param queryContext the data and resources needed to perform the query * @param searchAfter indicates where Elasticsearch should continue querying on subsequent requests, if at all * @param size the minimum size of the matches to chunk @@ -24,29 +20,14 @@ export const findPotentialMatches = async ( size: number ) => { const queryResult = await query(queryContext, searchAfter, size); - const checkGroups = new Set(); const monitorIds: string[] = []; get(queryResult, 'aggregations.monitors.buckets', []).forEach((b: any) => { const monitorId = b.key.monitor_id; monitorIds.push(monitorId); - - // Doc count can be zero if status filter optimization does not match - if (b.doc_count > 0) { - // Here we grab the most recent 2 check groups per location and add them to the list. - // Why 2? Because the most recent one may be a partial result from mode: all, and hence not match a summary doc. - b.locations.buckets.forEach((lb: any) => { - lb.ips.buckets.forEach((ib: any) => { - ib.top.hits.hits.forEach((h: any) => { - checkGroups.add(h._source.monitor.check_group); - }); - }); - }); - } }); return { monitorIds, - checkGroups, searchAfter: queryResult.aggregations?.monitors?.after_key, }; }; @@ -89,29 +70,6 @@ const queryBody = async (queryContext: QueryContext, searchAfter: any, size: num }, ], }, - aggs: { - // Here we grab the most recent 2 check groups per location. - // Why 2? Because the most recent one may not be for a summary, it may be incomplete. - locations: { - terms: { field: 'observer.geo.name', missing: '__missing__' }, - aggs: { - ips: { - terms: { field: 'monitor.ip', missing: '0.0.0.0' }, - aggs: { - top: { - top_hits: { - sort: [{ '@timestamp': 'desc' }], - _source: { - includes: ['monitor.check_group', '@timestamp'], - }, - size: 2, - }, - }, - }, - }, - }, - }, - }, }, }, }; diff --git a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts index e5e3de322cbc7e..2f54f3f6dd689f 100644 --- a/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts +++ b/x-pack/plugins/uptime/server/lib/requests/search/refine_potential_matches.ts @@ -18,18 +18,14 @@ import { MonitorGroups, MonitorLocCheckGroup } from './fetch_page'; // check groups for their associated monitor IDs. If not, it discards the result. export const refinePotentialMatches = async ( queryContext: QueryContext, - potentialMatchMonitorIDs: string[], - potentialMatchCheckGroups: Set + potentialMatchMonitorIDs: string[] ): Promise => { if (potentialMatchMonitorIDs.length === 0) { return []; } - const recentGroupsMatchingStatus = await fullyMatchingIds( - queryContext, - potentialMatchMonitorIDs, - potentialMatchCheckGroups - ); + const queryResult = await query(queryContext, potentialMatchMonitorIDs); + const recentGroupsMatchingStatus = await fullyMatchingIds(queryResult, queryContext.statusFilter); // Return the monitor groups filtering out potential matches that weren't current const matches: MonitorGroups[] = potentialMatchMonitorIDs @@ -49,27 +45,35 @@ export const refinePotentialMatches = async ( return matches; }; -const fullyMatchingIds = async ( - queryContext: QueryContext, - potentialMatchMonitorIDs: string[], - potentialMatchCheckGroups: Set -) => { - const mostRecentQueryResult = await mostRecentCheckGroups(queryContext, potentialMatchMonitorIDs); - +export const fullyMatchingIds = (queryResult: any, statusFilter?: string) => { const matching = new Map(); - MonitorLoop: for (const monBucket of mostRecentQueryResult.aggregations.monitor.buckets) { + MonitorLoop: for (const monBucket of queryResult.aggregations.monitor.buckets) { const monitorId: string = monBucket.key; const groups: MonitorLocCheckGroup[] = []; + // Did at least one location match? + let matched = false; for (const locBucket of monBucket.location.buckets) { const location = locBucket.key; - const topSource = locBucket.top.hits.hits[0]._source; - const checkGroup = topSource.monitor.check_group; - const status = topSource.summary.down > 0 ? 'down' : 'up'; + const latestSource = locBucket.summaries.latest.hits.hits[0]._source; + const latestStillMatchingSource = locBucket.latest_matching.top.hits.hits[0]?._source; + // If the most recent document still matches the most recent document matching the current filters + // we can include this in the result + // + // We just check if the timestamp is greater. Note this may match an incomplete check group + // that has not yet sent a summary doc + if ( + latestStillMatchingSource && + latestStillMatchingSource['@timestamp'] >= latestSource['@timestamp'] + ) { + matched = true; + } + const checkGroup = latestSource.monitor.check_group; + const status = latestSource.summary.down > 0 ? 'down' : 'up'; // This monitor doesn't match, so just skip ahead and don't add it to the output // Only skip in case of up statusFilter, for a monitor to be up, all checks should be up - if (queryContext?.statusFilter === 'up' && queryContext.statusFilter !== status) { + if (statusFilter === 'up' && statusFilter !== status) { continue MonitorLoop; } @@ -78,12 +82,12 @@ const fullyMatchingIds = async ( location, checkGroup, status, - summaryTimestamp: topSource['@timestamp'], + summaryTimestamp: new Date(latestSource['@timestamp']), }); } - // We only truly match the monitor if one of the most recent check groups was found in the potential matches phase - if (groups.some((g) => potentialMatchCheckGroups.has(g.checkGroup))) { + // If one location matched, include data from all locations in the result set + if (matched) { matching.set(monitorId, groups); } } @@ -91,7 +95,7 @@ const fullyMatchingIds = async ( return matching; }; -export const mostRecentCheckGroups = async ( +export const query = async ( queryContext: QueryContext, potentialMatchMonitorIDs: string[] ): Promise => { @@ -104,8 +108,6 @@ export const mostRecentCheckGroups = async ( filter: [ await queryContext.dateRangeFilter(), { terms: { 'monitor.id': potentialMatchMonitorIDs } }, - // only match summary docs because we only want the latest *complete* check group. - { exists: { field: 'summary' } }, ], }, }, @@ -116,13 +118,39 @@ export const mostRecentCheckGroups = async ( location: { terms: { field: 'observer.geo.name', missing: 'N/A', size: 100 }, aggs: { - top: { - top_hits: { - sort: [{ '@timestamp': 'desc' }], - _source: { - includes: ['monitor.check_group', '@timestamp', 'summary.up', 'summary.down'], + summaries: { + // only match summary docs because we only want the latest *complete* check group. + filter: { exists: { field: 'summary' } }, + aggs: { + latest: { + top_hits: { + sort: [{ '@timestamp': 'desc' }], + _source: { + includes: [ + 'monitor.check_group', + '@timestamp', + 'summary.up', + 'summary.down', + ], + }, + size: 1, + }, + }, + }, + }, + // We want to find the latest check group, even if it's not part of a summary + latest_matching: { + filter: queryContext.filterClause || { match_all: {} }, + aggs: { + top: { + top_hits: { + sort: [{ '@timestamp': 'desc' }], + _source: { + includes: ['monitor.check_group', '@timestamp'], + }, + size: 1, + }, }, - size: 1, }, }, },