From 5c13594230ad5036af292a95720bbdaa440fb0c0 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 13 Apr 2021 20:52:17 -0700 Subject: [PATCH] [Fleet] Improve performance of data stream API (#97058) * Improve performance of data stream API * Remove extra logger, replace filter with reduce * Remove unused import # Conflicts: # x-pack/plugins/fleet/server/routes/data_streams/handlers.ts # x-pack/plugins/fleet/server/services/epm/packages/get.ts --- .../server/routes/data_streams/handlers.ts | 83 ++++++++++++------- .../fleet/server/services/epm/packages/get.ts | 10 +-- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts index 4424fc478c682f..5f232fc3768335 100644 --- a/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/data_streams/handlers.ts @@ -5,11 +5,12 @@ * 2.0. */ -import { RequestHandler, SavedObjectsClientContract } from 'src/core/server'; +import { RequestHandler, SavedObjectsBulkGetObject } from 'src/core/server'; import { keyBy, keys, merge } from 'lodash'; import { DataStream } from '../../types'; -import { GetDataStreamsResponse, KibanaAssetType, KibanaSavedObjectType } from '../../../common'; -import { getPackageSavedObjects, getKibanaSavedObject } from '../../services/epm/packages/get'; +import { GetDataStreamsResponse, KibanaSavedObjectType } from '../../../common'; +import { getPackageSavedObjects } from '../../services/epm/packages/get'; + import { defaultIngestErrorHandler } from '../../errors'; const DATA_STREAM_INDEX_PATTERN = 'logs-*-*,metrics-*-*,traces-*-*'; @@ -80,6 +81,40 @@ export const getListHandler: RequestHandler = async (context, request, response) const packageSavedObjectsByName = keyBy(packageSavedObjects.saved_objects, 'id'); const packageMetadata: any = {}; + // Get dashboard information for all packages + const dashboardIdsByPackageName = packageSavedObjects.saved_objects.reduce< + Record + >((allDashboards, pkgSavedObject) => { + const dashboards: string[] = []; + (pkgSavedObject.attributes?.installed_kibana || []).forEach((o) => { + if (o.type === KibanaSavedObjectType.dashboard) { + dashboards.push(o.id); + } + }); + allDashboards[pkgSavedObject.id] = dashboards; + return allDashboards; + }, {}); + const allDashboardSavedObjects = await context.core.savedObjects.client.bulkGet<{ + title?: string; + }>( + Object.values(dashboardIdsByPackageName).reduce( + (allDashboards, dashboardIds) => { + return allDashboards.concat( + dashboardIds.map((id) => ({ + id, + type: KibanaSavedObjectType.dashboard, + fields: ['title'], + })) + ); + }, + [] + ) + ); + const allDashboardSavedObjectsById = keyBy( + allDashboardSavedObjects.saved_objects, + (dashboardSavedObject) => dashboardSavedObject.id + ); + // Query additional information for each data stream const dataStreamPromises = dataStreamNames.map(async (dataStreamName) => { const dataStream = dataStreams[dataStreamName]; @@ -157,19 +192,23 @@ export const getListHandler: RequestHandler = async (context, request, response) // - and we didn't pick the metadata in an earlier iteration of this map() if (!packageMetadata[pkgName]) { // then pick the dashboards from the package saved object - const dashboards = - pkgSavedObject.attributes?.installed_kibana?.filter( - (o) => o.type === KibanaSavedObjectType.dashboard - ) || []; - // and then pick the human-readable titles from the dashboard saved objects - const enhancedDashboards = await getEnhancedDashboards( - context.core.savedObjects.client, - dashboards - ); + const packageDashboardIds = dashboardIdsByPackageName[pkgName] || []; + const packageDashboards = packageDashboardIds.reduce< + Array<{ id: string; title: string }> + >((dashboards, dashboardId) => { + const dashboard = allDashboardSavedObjectsById[dashboardId]; + if (dashboard) { + dashboards.push({ + id: dashboard.id, + title: dashboard.attributes.title || dashboard.id, + }); + } + return dashboards; + }, []); packageMetadata[pkgName] = { version: pkgSavedObject.attributes?.version || '', - dashboards: enhancedDashboards, + dashboards: packageDashboards, }; } @@ -194,21 +233,3 @@ export const getListHandler: RequestHandler = async (context, request, response) return defaultIngestErrorHandler({ error, response }); } }; - -const getEnhancedDashboards = async ( - savedObjectsClient: SavedObjectsClientContract, - dashboards: any[] -) => { - const dashboardsPromises = dashboards.map(async (db) => { - const dbSavedObject: any = await getKibanaSavedObject( - savedObjectsClient, - KibanaAssetType.dashboard, - db.id - ); - return { - id: db.id, - title: dbSavedObject.attributes?.title || db.id, - }; - }); - return await Promise.all(dashboardsPromises); -}; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index b6d10178caacfc..3a482db122656a 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -15,7 +15,7 @@ import { } from '../../../../common'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { ArchivePackage, RegistryPackage, EpmPackageAdditions } from '../../../../common/types'; -import { Installation, PackageInfo, KibanaAssetType } from '../../../types'; +import { Installation, PackageInfo } from '../../../types'; import { IngestManagerError } from '../../../errors'; import { appContextService } from '../../'; import * as Registry from '../registry'; @@ -254,11 +254,3 @@ function sortByName(a: { name: string }, b: { name: string }) { return 0; } } - -export async function getKibanaSavedObject( - savedObjectsClient: SavedObjectsClientContract, - type: KibanaAssetType, - id: string -) { - return savedObjectsClient.get(type, id); -}