From 57f0e8c4a2c0d8d250f331864bad644c27d789ce Mon Sep 17 00:00:00 2001 From: Joseph Crail Date: Tue, 19 Apr 2022 09:25:51 -0700 Subject: [PATCH] Add stackframe metadata to TopN stack trace response (#64) * Refactor grouping metadata by stacktraces * Collate stacktrace events same as flamegraph * Transform raw aggregation result We now use the same TopN histogram bucket format used by prodfiler. * Return no hits when querying stacktrace events * Simplify topN query for stacktrace events * Reduce query time by avoiding unneeded searches * Refactor out query for stacktraces * Refactor out queries for frames and executables * Convert Elastic response to stackframe metadata * Search for frames and executables in parallel * Add default value for contexts * Add types for function parameters * Fix compiler warnings and failing tests --- .../elasticsearch/client/configure_client.ts | 36 +++ src/plugins/profiling/common/flamegraph.ts | 38 +-- src/plugins/profiling/common/index.ts | 28 +- src/plugins/profiling/common/profiling.ts | 34 ++ src/plugins/profiling/common/topn.ts | 54 ++++ .../FlameGraph/helpers/BitmapTextEllipse.ts | 2 +- .../helpers/binarySearchLowerLimit.ts | 2 +- .../FlameGraph/helpers/debounce.ts | 2 +- .../FlameGraph/helpers/lifecycle.ts | 2 +- .../FlameGraph/helpers/regexCreator.ts | 2 +- .../FlameGraph/services/featureFlags.ts | 2 +- .../public/components/chart-grid.tsx | 4 +- .../public/components/contexts/flamegraph.tsx | 2 +- .../public/components/contexts/topn.tsx | 2 +- .../public/components/flamegraph.tsx | 2 +- .../public/components/stacktrace-nav.tsx | 4 +- .../server/routes/flamechart.test.ts | 24 +- .../profiling/server/routes/flamechart.ts | 285 ++--------------- .../profiling/server/routes/mappings.ts | 34 +- .../server/routes/stacktrace.test.ts | 31 ++ .../profiling/server/routes/stacktrace.ts | 294 ++++++++++++++++++ .../profiling/server/routes/topn.test.ts | 27 +- src/plugins/profiling/server/routes/topn.ts | 88 +++--- 23 files changed, 570 insertions(+), 429 deletions(-) create mode 100644 src/plugins/profiling/common/topn.ts create mode 100644 src/plugins/profiling/server/routes/stacktrace.test.ts create mode 100644 src/plugins/profiling/server/routes/stacktrace.ts diff --git a/src/core/server/elasticsearch/client/configure_client.ts b/src/core/server/elasticsearch/client/configure_client.ts index 020f3d66a42a5e..0d543cb0352019 100644 --- a/src/core/server/elasticsearch/client/configure_client.ts +++ b/src/core/server/elasticsearch/client/configure_client.ts @@ -8,6 +8,9 @@ import { Client, HttpConnection } from '@elastic/elasticsearch'; import type { Logger } from '@kbn/logging'; +import type { KibanaClient } from '@elastic/elasticsearch/lib/api/kibana'; +import agent, { Span } from 'elastic-apm-node'; +import LRUCache from 'lru-cache'; import { parseClientOptions, ElasticsearchClientConfig } from './client_config'; import { instrumentEsQueryAndDeprecationLogger } from './log_query_and_deprecation'; import { createTransport } from './create_transport'; @@ -31,12 +34,45 @@ export const configureClient = ( const clientOptions = parseClientOptions(config, scoped); const KibanaTransport = createTransport({ getExecutionContext }); + const cache = new LRUCache({ + max: 100, + }); + const client = new Client({ ...clientOptions, Transport: KibanaTransport, Connection: HttpConnection, }); + function startSpan(name: string) { + const span = agent.startSpan(name, 'db', 'elasticsearch', { exitSpan: true }); + return span; + } + + client.diagnostic.on('serialization', (err, result) => { + if (!err) { + cache.set(result?.meta.request.id, startSpan('serialization')); + } + }); + + client.diagnostic.on('request', (err, result) => { + cache.get(result?.meta.request.id)?.end(); + if (!err) { + cache.set(result?.meta.request.id, startSpan('request')); + } + }); + + client.diagnostic.on('deserialization', (err, result) => { + cache.get(result?.requestId)?.end(); + if (!err) { + cache.set(result?.requestId, startSpan('deserialization')); + } + }); + + client.diagnostic.on('response', (err, result) => { + cache.get(result?.meta.request.id)?.end(); + }); + instrumentEsQueryAndDeprecationLogger({ logger, client, type }); return client; diff --git a/src/plugins/profiling/common/flamegraph.ts b/src/plugins/profiling/common/flamegraph.ts index 1e92d1495c0ae3..8ed0497fe6b847 100644 --- a/src/plugins/profiling/common/flamegraph.ts +++ b/src/plugins/profiling/common/flamegraph.ts @@ -19,7 +19,7 @@ import { StackFrame, Executable, createStackFrameMetadata, - StackFrameMetadata, + groupStackFrameMetadataByStackTrace, } from './profiling'; interface PixiFlameGraph extends CallerCalleeNode { @@ -75,36 +75,6 @@ export class FlameGraph { this.logger = logger; } - // getFrameMetadataForTraces collects all of the per-stack-frame metadata for a - // given set of trace IDs and their respective stack frames. - // - // This is similar to GetTraceMetaData in pf-storage-backend/storagebackend/storagebackendv1/reads_webservice.go - private getFrameMetadataForTraces(): Map { - const frameMetadataForTraces = new Map(); - for (const [stackTraceID, trace] of this.stacktraces) { - const frameMetadata = new Array(); - for (let i = 0; i < trace.FrameID.length; i++) { - const frame = this.stackframes.get(trace.FrameID[i])!; - const executable = this.executables.get(trace.FileID[i])!; - - const metadata = createStackFrameMetadata({ - FileID: Buffer.from(trace.FileID[i], 'base64url').toString('hex'), - FrameType: trace.Type[i], - AddressOrLine: frame.LineNumber, - FunctionName: frame.FunctionName, - FunctionOffset: frame.FunctionOffset, - SourceLine: frame.LineNumber, - ExeFileName: executable.FileName, - Index: i, - }); - - frameMetadata.push(metadata); - } - frameMetadataForTraces.set(stackTraceID, frameMetadata); - } - return frameMetadataForTraces; - } - private getExeFileName(exe: any, type: number) { if (exe?.FileName === undefined) { this.logger.warn('missing executable FileName'); @@ -188,7 +158,11 @@ export class FlameGraph { toPixi(): PixiFlameGraph { const rootFrame = createStackFrameMetadata(); - const frameMetadataForTraces = this.getFrameMetadataForTraces(); + const frameMetadataForTraces = groupStackFrameMetadataByStackTrace( + this.stacktraces, + this.stackframes, + this.executables + ); const diagram = createCallerCalleeIntermediateRoot( rootFrame, this.events, diff --git a/src/plugins/profiling/common/index.ts b/src/plugins/profiling/common/index.ts index 469243ccae0550..504e667e597399 100644 --- a/src/plugins/profiling/common/index.ts +++ b/src/plugins/profiling/common/index.ts @@ -28,20 +28,10 @@ function toMilliseconds(seconds: string): number { return parseInt(seconds, 10) * 1000; } -export function getTopN(obj) { +export function getTopN(obj: any) { const data = []; - if (obj.topN?.histogram?.buckets!) { - // needed for data served from Elasticsearch - for (let i = 0; i < obj.topN.histogram.buckets.length; i++) { - const bucket = obj.topN.histogram.buckets[i]; - for (let j = 0; j < bucket.group_by.buckets.length; j++) { - const v = bucket.group_by.buckets[j]; - data.push({ x: bucket.key, y: v.Count.value, g: v.key }); - } - } - } else if (obj.TopN!) { - // needed for data served from fixtures + if (obj.TopN!) { for (const x in obj.TopN) { if (obj.TopN.hasOwnProperty(x)) { const values = obj.TopN[x]; @@ -56,7 +46,7 @@ export function getTopN(obj) { return data; } -export function groupSamplesByCategory(samples) { +export function groupSamplesByCategory(samples: any) { const series = new Map(); for (let i = 0; i < samples.length; i++) { const v = samples[i]; @@ -74,3 +64,15 @@ export function timeRangeFromRequest(request: any): [number, number] { const timeTo = parseInt(request.query.timeTo!, 10); return [timeFrom, timeTo]; } + +// Converts from a Map object to a Record object since Map objects are not +// serializable to JSON by default +export function fromMapToRecord(m: Map): Record { + let output: Record = {}; + + for (const [key, value] of m) { + output[key] = value; + } + + return output; +} diff --git a/src/plugins/profiling/common/profiling.ts b/src/plugins/profiling/common/profiling.ts index f7d05683d5af05..46f83b84f206ce 100644 --- a/src/plugins/profiling/common/profiling.ts +++ b/src/plugins/profiling/common/profiling.ts @@ -96,6 +96,40 @@ export function createStackFrameMetadata( return metadata; } +// groupStackFrameMetadataByStackTrace collects all of the per-stack-frame +// metadata for a given set of trace IDs and their respective stack frames. +// +// This is similar to GetTraceMetaData in pf-storage-backend/storagebackend/storagebackendv1/reads_webservice.go +export function groupStackFrameMetadataByStackTrace( + stackTraces: Map, + stackFrames: Map, + executables: Map +): Map { + const frameMetadataForTraces = new Map(); + for (const [stackTraceID, trace] of stackTraces) { + const frameMetadata = new Array(); + for (let i = 0; i < trace.FrameID.length; i++) { + const frame = stackFrames.get(trace.FrameID[i])!; + const executable = executables.get(trace.FileID[i])!; + + const metadata = createStackFrameMetadata({ + FileID: Buffer.from(trace.FileID[i], 'base64url').toString('hex'), + FrameType: trace.Type[i], + AddressOrLine: frame.LineNumber, + FunctionName: frame.FunctionName, + FunctionOffset: frame.FunctionOffset, + SourceLine: frame.LineNumber, + ExeFileName: executable.FileName, + Index: i, + }); + + frameMetadata.push(metadata); + } + frameMetadataForTraces.set(stackTraceID, frameMetadata); + } + return frameMetadataForTraces; +} + export type FrameGroup = Pick< StackFrameMetadata, 'FileID' | 'ExeFileName' | 'FunctionName' | 'AddressOrLine' | 'SourceFilename' diff --git a/src/plugins/profiling/common/topn.ts b/src/plugins/profiling/common/topn.ts new file mode 100644 index 00000000000000..b11ec56346f0cb --- /dev/null +++ b/src/plugins/profiling/common/topn.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + AggregationsHistogramAggregate, + AggregationsHistogramBucket, +} from '@elastic/elasticsearch/lib/api/types'; + +import { StackFrameMetadata } from './profiling'; + +type TopNBucket = { + Value: string; + Count: number; +}; + +type TopNBucketsByDate = { + TopN: Record; +}; + +type TopNContainers = TopNBucketsByDate; +type TopNDeployments = TopNBucketsByDate; +type TopNHosts = TopNBucketsByDate; +type TopNThreads = TopNBucketsByDate; + +type TopNTraces = TopNBucketsByDate & { + Metadata: Record; +}; + +type TopN = TopNContainers | TopNDeployments | TopNHosts | TopNThreads | TopNTraces; + +export function createTopNBucketsByDate( + histogram: AggregationsHistogramAggregate +): TopNBucketsByDate { + const topNBucketsByDate: Record = {}; + + const histogramBuckets = (histogram?.buckets as AggregationsHistogramBucket[]) ?? []; + for (let i = 0; i < histogramBuckets.length; i++) { + const key = histogramBuckets[i].key / 1000; + topNBucketsByDate[key] = []; + histogramBuckets[i].group_by.buckets.forEach((item: any) => { + topNBucketsByDate[key].push({ + Value: item.key, + Count: item.count.value, + }); + }); + } + + return { TopN: topNBucketsByDate }; +} diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/BitmapTextEllipse.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/BitmapTextEllipse.ts index 09329f3b6633aa..4454514ffbdfff 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/BitmapTextEllipse.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/BitmapTextEllipse.ts @@ -95,4 +95,4 @@ export class BitmapTextEllipse extends Pixi.BitmapText { */ this.dirty = true } -} \ No newline at end of file +} diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/binarySearchLowerLimit.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/binarySearchLowerLimit.ts index 62ee6e259a0b6e..9400e10b1b2e22 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/binarySearchLowerLimit.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/binarySearchLowerLimit.ts @@ -30,4 +30,4 @@ export const binarySearchLowerLimit = ( } return min; -} \ No newline at end of file +} diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/debounce.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/debounce.ts index b2a20478f1867d..ad49c5abd53f0b 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/debounce.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/debounce.ts @@ -8,4 +8,4 @@ const debounce = (callback: Function, wait: number) => { }; } -export default debounce \ No newline at end of file +export default debounce diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/lifecycle.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/lifecycle.ts index 1973a18a7c9427..2b067704788df2 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/lifecycle.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/lifecycle.ts @@ -151,4 +151,4 @@ export const useResizeListenerEffect = ( callback(canvas, renderer, viewport) }, 500) }, [gameCanvasRef, sidebar, renderer, viewport, callback]) -} \ No newline at end of file +} diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/regexCreator.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/regexCreator.ts index a88947f5e02ed4..f70ebc1e86693f 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/regexCreator.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/helpers/regexCreator.ts @@ -6,4 +6,4 @@ export const safeRegexCreator = (pattern: string, flags?: string | undefined): R // in the future this could be handled by the UI and we could show a message to the user return null } -} \ No newline at end of file +} diff --git a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/services/featureFlags.ts b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/services/featureFlags.ts index 80c17bd314828f..007034509012b5 100644 --- a/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/services/featureFlags.ts +++ b/src/plugins/profiling/public/components/PixiFlamechart/FlameGraph/services/featureFlags.ts @@ -15,4 +15,4 @@ export const toggleSandboxMode = () => { } else { setSandboxModeTo(ENABLED_KEY) } -} \ No newline at end of file +} diff --git a/src/plugins/profiling/public/components/chart-grid.tsx b/src/plugins/profiling/public/components/chart-grid.tsx index d1048300ebffa6..1f6cf10bc0af25 100644 --- a/src/plugins/profiling/public/components/chart-grid.tsx +++ b/src/plugins/profiling/public/components/chart-grid.tsx @@ -26,8 +26,8 @@ export interface ChartGridProps { export const ChartGrid: React.FC = ({ maximum }) => { const ctx = useContext(TopNContext); - const printSubCharts = (series) => { - let keys = Array.from(series.keys()); + const printSubCharts = (series: any) => { + let keys: string[] = Array.from(series.keys()); const ncharts = Math.min(maximum, series.size); keys = keys.slice(0, ncharts); diff --git a/src/plugins/profiling/public/components/contexts/flamegraph.tsx b/src/plugins/profiling/public/components/contexts/flamegraph.tsx index 7eaf2b7730966e..70630e6f91972b 100644 --- a/src/plugins/profiling/public/components/contexts/flamegraph.tsx +++ b/src/plugins/profiling/public/components/contexts/flamegraph.tsx @@ -8,4 +8,4 @@ import { createContext } from 'react'; -export const FlameGraphContext = createContext(); +export const FlameGraphContext = createContext({}); diff --git a/src/plugins/profiling/public/components/contexts/topn.tsx b/src/plugins/profiling/public/components/contexts/topn.tsx index c4f7c398b20c52..5ea936707fbcac 100644 --- a/src/plugins/profiling/public/components/contexts/topn.tsx +++ b/src/plugins/profiling/public/components/contexts/topn.tsx @@ -8,4 +8,4 @@ import { createContext } from 'react'; -export const TopNContext = createContext(); +export const TopNContext = createContext({}); diff --git a/src/plugins/profiling/public/components/flamegraph.tsx b/src/plugins/profiling/public/components/flamegraph.tsx index 0b81ceb06c0cfe..cec7f88abd9965 100644 --- a/src/plugins/profiling/public/components/flamegraph.tsx +++ b/src/plugins/profiling/public/components/flamegraph.tsx @@ -32,7 +32,7 @@ export const FlameGraph: React.FC = ({ id, height }) => { } const { leaves } = ctx; - const maxDepth = Math.max(...leaves.map((node) => node.depth)); + const maxDepth = Math.max(...leaves.map((node: any) => node.depth)); const result = [...new Array(maxDepth)].map((_, depth) => { return { diff --git a/src/plugins/profiling/public/components/stacktrace-nav.tsx b/src/plugins/profiling/public/components/stacktrace-nav.tsx index 611cd43d5d94e6..670bc9b0dddfc8 100644 --- a/src/plugins/profiling/public/components/stacktrace-nav.tsx +++ b/src/plugins/profiling/public/components/stacktrace-nav.tsx @@ -45,7 +45,7 @@ export const StackTraceNavigation = ({ fetchTopN, setTopN }) => { const [toggleTopNSelected, setToggleTopNSelected] = useState(`${topnButtonGroupPrefix}__0`); - const onTopNChange = (optionId) => { + const onTopNChange = (optionId: string) => { if (optionId === toggleTopNSelected) { return; } @@ -84,7 +84,7 @@ export const StackTraceNavigation = ({ fetchTopN, setTopN }) => { const [toggleDateSelected, setToggleDateSelected] = useState(`${dateButtonGroupPrefix}__0`); - const onDateChange = (optionId) => { + const onDateChange = (optionId: string) => { if (optionId === toggleDateSelected) { return; } diff --git a/src/plugins/profiling/server/routes/flamechart.test.ts b/src/plugins/profiling/server/routes/flamechart.test.ts index 2fee0605d4a934..6e74a36766688a 100644 --- a/src/plugins/profiling/server/routes/flamechart.test.ts +++ b/src/plugins/profiling/server/routes/flamechart.test.ts @@ -7,7 +7,7 @@ */ import { DownsampledEventsIndex, getSampledTraceEventsIndex } from './downsampling'; -import { extractFileIDFromFrameID, parallelMget } from './flamechart'; +import { parallelMget } from './flamechart'; import { ElasticsearchClient } from 'kibana/server'; describe('Using down-sampled indexes', () => { @@ -63,28 +63,6 @@ describe('Using down-sampled indexes', () => { }); }); -describe('Extract FileID from FrameID', () => { - test('extractFileIDFromFrameID', () => { - const tests: Array<{ - frameID: string; - expected: string; - }> = [ - { - frameID: 'aQpJmTLWydNvOapSFZOwKgAAAAAAB924', - expected: 'aQpJmTLWydNvOapSFZOwKg==', - }, - { - frameID: 'hz_u-HGyrN6qeIk6UIJeCAAAAAAAAAZZ', - expected: 'hz_u-HGyrN6qeIk6UIJeCA==', - }, - ]; - - for (const t of tests) { - expect(extractFileIDFromFrameID(t.frameID)).toEqual(t.expected); - } - }); -}); - describe('Calling mget from events to stacktraces', () => { test('parallel queries to ES are resolved as promises', async () => { const numberOfFrames = 4; diff --git a/src/plugins/profiling/server/routes/flamechart.ts b/src/plugins/profiling/server/routes/flamechart.ts index 0f1d0482b6d6f4..e39451f33c8261 100644 --- a/src/plugins/profiling/server/routes/flamechart.ts +++ b/src/plugins/profiling/server/routes/flamechart.ts @@ -7,76 +7,14 @@ */ import { schema } from '@kbn/config-schema'; import type { ElasticsearchClient, IRouter, Logger } from 'kibana/server'; -import { chunk } from 'lodash'; -import LRUCache from 'lru-cache'; import type { DataRequestHandlerContext } from '../../../data/server'; import { getRoutePaths } from '../../common'; import { FlameGraph } from '../../common/flamegraph'; -import { - Executable, - FileID, - StackFrame, - StackFrameID, - StackTrace, - StackTraceID, -} from '../../common/profiling'; +import { StackTraceID } from '../../common/profiling'; import { logExecutionLatency } from './logger'; import { newProjectTimeQuery, ProjectTimeQuery } from './mappings'; import { downsampleEventsRandomly, findDownsampledIndex } from './downsampling'; - -const traceLRU = new LRUCache({ max: 20000 }); -const frameIDToFileIDCache = new LRUCache({ max: 100000 }); - -// convertFrameIDToFileID extracts the FileID from the FrameID and returns as base64url string. -export function extractFileIDFromFrameID(frameID: string): string { - const fileIDChunk = frameID.slice(0, 23); - let fileID = frameIDToFileIDCache.get(fileIDChunk) as string; - if (fileID) return fileID; - - // Step 1: Convert the base64-encoded frameID to an array of 22 bytes. - // We use 'base64url' instead of 'base64' because frameID is encoded URL-friendly. - // The first 16 bytes contain the FileID. - const buf = Buffer.from(fileIDChunk, 'base64url'); - - // Convert the FileID bytes into base64 with URL-friendly encoding. - // We have to manually append '==' since we use the FileID string for - // comparing / looking up the FileID strings in the ES indices, which have - // the '==' appended. - // We may want to remove '==' in the future to reduce the uncompressed storage size by 10%. - fileID = buf.toString('base64url', 0, 16) + '=='; - frameIDToFileIDCache.set(fileIDChunk, fileID); - return fileID; -} - -// extractFileIDArrayFromFrameIDArray extracts all FileIDs from the array of FrameIDs -// and returns them as an array of base64url encoded strings. The order of this array -// corresponds to the order of the input array. -function extractFileIDArrayFromFrameIDArray(frameIDs: string[]): string[] { - const fileIDs = Array(frameIDs.length); - for (let i = 0; i < frameIDs.length; i++) { - fileIDs[i] = extractFileIDFromFrameID(frameIDs[i]); - } - return fileIDs; -} - -function getNumberOfUniqueStacktracesWithoutLeafNode( - stackTraces: Map, - level: number -): number { - // Calculate the reduction in lookups that would derive from - // StackTraces without leaf frame. - const stackTracesNoLeaf = new Set(); - for (const trace of stackTraces.values()) { - stackTracesNoLeaf.add( - JSON.stringify({ - FileID: trace.FileID.slice(level), - FrameID: trace.FrameID.slice(level), - Type: trace.Type.slice(level), - }) - ); - } - return stackTracesNoLeaf.size; -} +import { mgetExecutables, mgetStackFrames, mgetStackTraces, searchStackTraces } from './stacktrace'; export function parallelMget( nQueries: number, @@ -176,10 +114,9 @@ async function queryFlameGraph( ); let totalCount: number = resEvents.body.aggregations?.total_count.value; - let stackTraceEvents: Map; + let stackTraceEvents = new Map(); await logExecutionLatency(logger, 'processing events data', async () => { - stackTraceEvents = new Map(); resEvents.body.aggregations?.group_by.buckets.forEach((item: any) => { const traceid: StackTraceID = item.key.traceid; stackTraceEvents.set(traceid, item.count.value); @@ -200,206 +137,22 @@ async function queryFlameGraph( } // profiling-stacktraces is configured with 16 shards - const nQueries = 1; - const stackTraces = new Map(); - const stackFrameDocIDs = new Set(); // Set of unique FrameIDs - const executableDocIDs = new Set(); // Set of unique executable FileIDs. - const stackTraceIDs = [...stackTraceEvents.keys()]; - const chunkSize = Math.floor(stackTraceEvents.size / nQueries); - let chunks = chunk(stackTraceIDs, chunkSize); - - if (chunks.length !== nQueries) { - // The last array element contains the remainder, just drop it as irrelevant. - chunks = chunks.slice(0, nQueries); - } - - const stackResponses = await logExecutionLatency( - logger, - (testing ? 'search' : 'mget') + ' query for ' + stackTraceEvents.size + ' stacktraces', - async () => { - return await Promise.all( - chunks.map((ids) => { - if (testing) { - return client.search( - { - index: 'profiling-stacktraces', - size: stackTraceEvents.size, - sort: '_doc', - query: { - ids: { - values: [...ids], - }, - }, - _source: false, - docvalue_fields: ['FrameID', 'Type'], - }, - { - querystring: { - filter_path: 'hits.hits._id,hits.hits.fields.FrameID,hits.hits.fields.Type', - pre_filter_shard_size: 1, - }, - } - ); - } else { - return client.mget({ - index: 'profiling-stacktraces', - ids, - realtime: false, - _source_includes: ['FrameID', 'Type'], - }); - } - }) - ); - } - ); - - let totalFrames = 0; - await logExecutionLatency(logger, 'processing data', async () => { - if (testing) { - const traces = stackResponses.flatMap((response) => response.body.hits.hits); - for (const trace of traces) { - const frameIDs = trace.fields.FrameID as string[]; - const fileIDs = extractFileIDArrayFromFrameIDArray(frameIDs); - stackTraces.set(trace._id, { - FileID: fileIDs, - FrameID: frameIDs, - Type: trace.fields.Type, - }); - for (const frameID of frameIDs) { - stackFrameDocIDs.add(frameID); - } - for (const fileID of fileIDs) { - executableDocIDs.add(fileID); - } - } - } else { - // flatMap() is significantly slower than an explicit for loop - for (const res of stackResponses) { - for (const trace of res.body.docs) { - // Sometimes we don't find the trace. - // This is due to ES delays writing (data is not immediately seen after write). - // Also, ES doesn't know about transactions. - if (trace.found) { - const traceid = trace._id as StackTraceID; - let stackTrace = traceLRU.get(traceid) as StackTrace; - if (!stackTrace) { - const frameIDs = trace._source.FrameID as string[]; - stackTrace = { - FileID: extractFileIDArrayFromFrameIDArray(frameIDs), - FrameID: frameIDs, - Type: trace._source.Type, - }; - traceLRU.set(traceid, stackTrace); - } - - totalFrames += stackTrace.FrameID.length; - stackTraces.set(traceid, stackTrace); - for (const frameID of stackTrace.FrameID) { - stackFrameDocIDs.add(frameID); - } - for (const fileID of stackTrace.FileID) { - executableDocIDs.add(fileID); - } - } - } - } - } - }); - - if (stackTraces.size !== 0) { - logger.info('Average size of stacktrace: ' + totalFrames / stackTraces.size); - } - - if (stackTraces.size < stackTraceEvents.size) { - logger.info( - 'failed to find ' + - (stackTraceEvents.size - stackTraces.size) + - ' stacktraces (todo: find out why)' - ); - } - - /* - logger.info( - '* unique stacktraces without leaf frame: ' + - getNumberOfUniqueStacktracesWithoutLeafNode(stackTraces, 1) - ); - - logger.info( - '* unique stacktraces without 2 leaf frames: ' + - getNumberOfUniqueStacktracesWithoutLeafNode(stackTraces, 2) - ); -*/ - - const resStackFrames = await logExecutionLatency( - logger, - 'mget query for ' + stackFrameDocIDs.size + ' stackframes', - async () => { - return await client.mget({ - index: 'profiling-stackframes', - ids: [...stackFrameDocIDs], - realtime: false, - }); - } - ); - - // Create a lookup map StackFrameID -> StackFrame. - const stackFrames = new Map(); - let framesFound = 0; - await logExecutionLatency(logger, 'processing data', async () => { - for (const frame of resStackFrames.body.docs) { - if (frame.found) { - stackFrames.set(frame._id, frame._source); - framesFound++; - } else { - stackFrames.set(frame._id, { - FileName: '', - FunctionName: '', - FunctionOffset: 0, - LineNumber: 0, - SourceType: 0, - }); - } - } - }); - logger.info('found ' + framesFound + ' / ' + stackFrameDocIDs.size + ' frames'); - - const resExecutables = await logExecutionLatency( - logger, - 'mget query for ' + executableDocIDs.size + ' executables', - async () => { - return await client.mget({ - index: 'profiling-executables', - ids: [...executableDocIDs], - _source_includes: ['FileName'], - }); - } - ); - - // Create a lookup map StackFrameID -> StackFrame. - const executables = new Map(); - await logExecutionLatency(logger, 'processing data', async () => { - for (const exe of resExecutables.body.docs) { - if (exe.found) { - executables.set(exe._id, exe._source); - } else { - executables.set(exe._id, { - FileName: '', - }); - } - } - }); - - return new Promise((resolve, _) => { - return resolve( - new FlameGraph( - eventsIndex.sampleRate, - totalCount, - stackTraceEvents, - stackTraces, - stackFrames, - executables, - logger - ) + const { stackTraces, stackFrameDocIDs, executableDocIDs } = testing + ? await searchStackTraces(logger, client, stackTraceEvents) + : await mgetStackTraces(logger, client, stackTraceEvents); + + return Promise.all([ + mgetStackFrames(logger, client, stackFrameDocIDs), + mgetExecutables(logger, client, executableDocIDs), + ]).then(([stackFrames, executables]) => { + return new FlameGraph( + eventsIndex.sampleRate, + totalCount, + stackTraceEvents, + stackTraces, + stackFrames, + executables, + logger ); }); } diff --git a/src/plugins/profiling/server/routes/mappings.ts b/src/plugins/profiling/server/routes/mappings.ts index 32b0333b54b3c7..770dd6c5c02a48 100644 --- a/src/plugins/profiling/server/routes/mappings.ts +++ b/src/plugins/profiling/server/routes/mappings.ts @@ -78,7 +78,7 @@ export function autoHistogramSumCountOnGroupByField( size: topNItems, }, aggs: { - Count: { + count: { sum: { field: 'Count', }, @@ -89,7 +89,7 @@ export function autoHistogramSumCountOnGroupByField( }; } -function getExeFileName(obj) { +function getExeFileName(obj: any) { if (obj.ExeFileName === undefined) { return ''; } @@ -120,17 +120,17 @@ function getExeFileName(obj) { } } -function checkIfStringHasParentheses(s) { +function checkIfStringHasParentheses(s: string) { return /\(|\)/.test(s); } -function getFunctionName(obj) { +function getFunctionName(obj: any) { return obj.FunctionName !== '' && !checkIfStringHasParentheses(obj.FunctionName) ? `${obj.FunctionName}()` : obj.FunctionName; } -function getBlockName(obj) { +function getBlockName(obj: any) { if (obj.FunctionName !== '') { const sourceFileName = obj.SourceFilename; const sourceURL = sourceFileName ? sourceFileName.split('/').pop() : ''; @@ -139,22 +139,22 @@ function getBlockName(obj) { return getExeFileName(obj); } -const sortFlamechartBySamples = function (a, b) { +const compareFlamechartSample = function (a: any, b: any) { return b.Samples - a.Samples; }; -const sortFlamechart = function (data) { - data.Callees.sort(sortFlamechartBySamples); +const sortFlamechart = function (data: any) { + data.Callees.sort(compareFlamechartSample); return data; }; -const parseFlamechart = function (data) { +const parseFlamechart = function (data: any) { const parsedData = sortFlamechart(data); parsedData.Callees = data.Callees.map(parseFlamechart); return parsedData; }; -function extendFlameGraph(node, depth) { +function extendFlameGraph(node: any, depth: any) { node.id = getBlockName(node); node.value = node.Samples; node.depth = depth; @@ -164,7 +164,7 @@ function extendFlameGraph(node, depth) { } } -function flattenTree(root, depth) { +function flattenTree(root: any, depth: any) { if (root.Callees.length === 0) { return [ { @@ -178,16 +178,16 @@ function flattenTree(root, depth) { ]; } - const children = root.Callees.flatMap((child) => flattenTree(child, depth + 1)); + const children = root.Callees.flatMap((child: any) => flattenTree(child, depth + 1)); - children.forEach((child) => { + children.forEach((child: any) => { child.pathFromRoot[depth] = root.id; }); return children; } -export function mapFlamechart(src) { +export function mapFlamechart(src: any) { src.ExeFileName = 'root'; const root = parseFlamechart(src); @@ -195,12 +195,6 @@ export function mapFlamechart(src) { extendFlameGraph(root, 0); const newRoot = flattenTree(root, 0); - [].map((node) => ({ - id: node.id, - value: node.value, - depth: node.depth, - pathFromRoot: node.pathFromRoot, - })); return { leaves: newRoot, diff --git a/src/plugins/profiling/server/routes/stacktrace.test.ts b/src/plugins/profiling/server/routes/stacktrace.test.ts new file mode 100644 index 00000000000000..d915de3ce21efd --- /dev/null +++ b/src/plugins/profiling/server/routes/stacktrace.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { extractFileIDFromFrameID } from './stacktrace'; + +describe('Extract FileID from FrameID', () => { + test('extractFileIDFromFrameID', () => { + const tests: Array<{ + frameID: string; + expected: string; + }> = [ + { + frameID: 'aQpJmTLWydNvOapSFZOwKgAAAAAAB924', + expected: 'aQpJmTLWydNvOapSFZOwKg==', + }, + { + frameID: 'hz_u-HGyrN6qeIk6UIJeCAAAAAAAAAZZ', + expected: 'hz_u-HGyrN6qeIk6UIJeCA==', + }, + ]; + + for (const t of tests) { + expect(extractFileIDFromFrameID(t.frameID)).toEqual(t.expected); + } + }); +}); diff --git a/src/plugins/profiling/server/routes/stacktrace.ts b/src/plugins/profiling/server/routes/stacktrace.ts new file mode 100644 index 00000000000000..bca19691dc46eb --- /dev/null +++ b/src/plugins/profiling/server/routes/stacktrace.ts @@ -0,0 +1,294 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { chunk } from 'lodash'; +import LRUCache from 'lru-cache'; +import type { ElasticsearchClient, Logger } from 'kibana/server'; +import { + Executable, + FileID, + StackFrame, + StackFrameID, + StackTrace, + StackTraceID, +} from '../../common/profiling'; +import { logExecutionLatency } from './logger'; + +const traceLRU = new LRUCache({ max: 20000 }); +const frameIDToFileIDCache = new LRUCache({ max: 100000 }); + +// convertFrameIDToFileID extracts the FileID from the FrameID and returns as base64url string. +export function extractFileIDFromFrameID(frameID: string): string { + const fileIDChunk = frameID.slice(0, 23); + let fileID = frameIDToFileIDCache.get(fileIDChunk) as string; + if (fileID) return fileID; + + // Step 1: Convert the base64-encoded frameID to an array of 22 bytes. + // We use 'base64url' instead of 'base64' because frameID is encoded URL-friendly. + // The first 16 bytes contain the FileID. + const buf = Buffer.from(fileIDChunk, 'base64url'); + + // Convert the FileID bytes into base64 with URL-friendly encoding. + // We have to manually append '==' since we use the FileID string for + // comparing / looking up the FileID strings in the ES indices, which have + // the '==' appended. + // We may want to remove '==' in the future to reduce the uncompressed storage size by 10%. + fileID = buf.toString('base64url', 0, 16) + '=='; + frameIDToFileIDCache.set(fileIDChunk, fileID); + return fileID; +} + +// extractFileIDArrayFromFrameIDArray extracts all FileIDs from the array of FrameIDs +// and returns them as an array of base64url encoded strings. The order of this array +// corresponds to the order of the input array. +function extractFileIDArrayFromFrameIDArray(frameIDs: string[]): string[] { + const fileIDs = Array(frameIDs.length); + for (let i = 0; i < frameIDs.length; i++) { + fileIDs[i] = extractFileIDFromFrameID(frameIDs[i]); + } + return fileIDs; +} + +export async function searchStackTraces( + logger: Logger, + client: ElasticsearchClient, + events: Map, + concurrency: number = 1 +) { + const stackTraceIDs = [...events.keys()]; + const chunkSize = Math.floor(events.size / concurrency); + let chunks = chunk(stackTraceIDs, chunkSize); + + if (chunks.length !== concurrency) { + // The last array element contains the remainder, just drop it as irrelevant. + chunks = chunks.slice(0, concurrency); + } + + const stackResponses = await logExecutionLatency( + logger, + 'search query for ' + events.size + ' stacktraces', + async () => { + return await Promise.all( + chunks.map((ids) => { + return client.search( + { + index: 'profiling-stacktraces', + size: events.size, + sort: '_doc', + query: { + ids: { + values: [...ids], + }, + }, + _source: false, + docvalue_fields: ['FrameID', 'Type'], + }, + { + querystring: { + filter_path: 'hits.hits._id,hits.hits.fields.FrameID,hits.hits.fields.Type', + pre_filter_shard_size: 1, + }, + } + ); + }) + ); + } + ); + + const stackTraces = new Map(); + const stackFrameDocIDs = new Set(); // Set of unique FrameIDs + const executableDocIDs = new Set(); // Set of unique executable FileIDs. + + await logExecutionLatency(logger, 'processing data', async () => { + const traces = stackResponses.flatMap((response) => response.body.hits.hits); + for (const trace of traces) { + const frameIDs = trace.fields.FrameID as string[]; + const fileIDs = extractFileIDArrayFromFrameIDArray(frameIDs); + stackTraces.set(trace._id, { + FileID: fileIDs, + FrameID: frameIDs, + Type: trace.fields.Type, + }); + for (const frameID of frameIDs) { + stackFrameDocIDs.add(frameID); + } + for (const fileID of fileIDs) { + executableDocIDs.add(fileID); + } + } + }); + + if (stackTraces.size < events.size) { + logger.info( + 'failed to find ' + (events.size - stackTraces.size) + ' stacktraces (todo: find out why)' + ); + } + + return { stackTraces, stackFrameDocIDs, executableDocIDs }; +} + +export async function mgetStackTraces( + logger: Logger, + client: ElasticsearchClient, + events: Map, + concurrency: number = 1 +) { + const stackTraceIDs = [...events.keys()]; + const chunkSize = Math.floor(events.size / concurrency); + let chunks = chunk(stackTraceIDs, chunkSize); + + if (chunks.length !== concurrency) { + // The last array element contains the remainder, just drop it as irrelevant. + chunks = chunks.slice(0, concurrency); + } + + const stackResponses = await logExecutionLatency( + logger, + 'mget query for ' + events.size + ' stacktraces', + async () => { + return await Promise.all( + chunks.map((ids) => { + return client.mget({ + index: 'profiling-stacktraces', + ids, + realtime: false, + _source_includes: ['FrameID', 'Type'], + }); + }) + ); + } + ); + + let totalFrames = 0; + const stackTraces = new Map(); + const stackFrameDocIDs = new Set(); // Set of unique FrameIDs + const executableDocIDs = new Set(); // Set of unique executable FileIDs. + + await logExecutionLatency(logger, 'processing data', async () => { + // flatMap() is significantly slower than an explicit for loop + for (const res of stackResponses) { + for (const trace of res.body.docs) { + // Sometimes we don't find the trace. + // This is due to ES delays writing (data is not immediately seen after write). + // Also, ES doesn't know about transactions. + if (trace.found) { + const traceid = trace._id as StackTraceID; + let stackTrace = traceLRU.get(traceid) as StackTrace; + if (!stackTrace) { + const frameIDs = trace._source.FrameID as string[]; + stackTrace = { + FileID: extractFileIDArrayFromFrameIDArray(frameIDs), + FrameID: frameIDs, + Type: trace._source.Type, + }; + traceLRU.set(traceid, stackTrace); + } + + totalFrames += stackTrace.FrameID.length; + stackTraces.set(traceid, stackTrace); + for (const frameID of stackTrace.FrameID) { + stackFrameDocIDs.add(frameID); + } + for (const fileID of stackTrace.FileID) { + executableDocIDs.add(fileID); + } + } + } + } + }); + + if (stackTraces.size !== 0) { + logger.info('Average size of stacktrace: ' + totalFrames / stackTraces.size); + } + + if (stackTraces.size < events.size) { + logger.info( + 'failed to find ' + (events.size - stackTraces.size) + ' stacktraces (todo: find out why)' + ); + } + + return { stackTraces, stackFrameDocIDs, executableDocIDs }; +} + +export async function mgetStackFrames( + logger: Logger, + client: ElasticsearchClient, + stackFrameIDs: Set +): Promise> { + const resStackFrames = await logExecutionLatency( + logger, + 'mget query for ' + stackFrameIDs.size + ' stackframes', + async () => { + return await client.mget({ + index: 'profiling-stackframes', + ids: [...stackFrameIDs], + realtime: false, + }); + } + ); + + // Create a lookup map StackFrameID -> StackFrame. + const stackFrames = new Map(); + let framesFound = 0; + await logExecutionLatency(logger, 'processing data', async () => { + const docs = resStackFrames.body?.docs ?? []; + for (const frame of docs) { + if (frame.found) { + stackFrames.set(frame._id, frame._source); + framesFound++; + } else { + stackFrames.set(frame._id, { + FileName: '', + FunctionName: '', + FunctionOffset: 0, + LineNumber: 0, + SourceType: 0, + }); + } + } + }); + + logger.info('found ' + framesFound + ' / ' + stackFrameIDs.size + ' frames'); + + return stackFrames; +} + +export async function mgetExecutables( + logger: Logger, + client: ElasticsearchClient, + executableIDs: Set +): Promise> { + const resExecutables = await logExecutionLatency( + logger, + 'mget query for ' + executableIDs.size + ' executables', + async () => { + return await client.mget({ + index: 'profiling-executables', + ids: [...executableIDs], + _source_includes: ['FileName'], + }); + } + ); + + // Create a lookup map StackFrameID -> StackFrame. + const executables = new Map(); + await logExecutionLatency(logger, 'processing data', async () => { + const docs = resExecutables.body?.docs ?? []; + for (const exe of docs) { + if (exe.found) { + executables.set(exe._id, exe._source); + } else { + executables.set(exe._id, { + FileName: '', + }); + } + } + }); + + return executables; +} diff --git a/src/plugins/profiling/server/routes/topn.test.ts b/src/plugins/profiling/server/routes/topn.test.ts index cc3bdd22ede34a..ae01e38bdcb5c7 100644 --- a/src/plugins/profiling/server/routes/topn.test.ts +++ b/src/plugins/profiling/server/routes/topn.test.ts @@ -34,31 +34,6 @@ describe('TopN data from Elasticsearch', () => { jest.clearAllMocks(); }); - describe('building the query', () => { - it('filters by projectID and aggregates timerange on histogram', async () => { - await topNElasticSearchQuery( - client, - logger, - index, - '123', - '456', - '789', - 200, - 'field', - kibanaResponseFactory - ); - expect(client.search).toHaveBeenCalledWith({ - index, - body: { - query: anyQuery, - aggs: { - histogram: testAgg, - }, - }, - }); - }); - }); - describe('when fetching Stack Traces', () => { it('should search first then mget', async () => { await topNElasticSearchQuery( @@ -73,7 +48,7 @@ describe('TopN data from Elasticsearch', () => { kibanaResponseFactory ); expect(client.search).toHaveBeenCalledTimes(2); - expect(client.mget).toHaveBeenCalledTimes(1); + expect(client.mget).toHaveBeenCalledTimes(2); }); }); }); diff --git a/src/plugins/profiling/server/routes/topn.ts b/src/plugins/profiling/server/routes/topn.ts index 1302e8adb13c57..63965dc20c2e0c 100644 --- a/src/plugins/profiling/server/routes/topn.ts +++ b/src/plugins/profiling/server/routes/topn.ts @@ -13,11 +13,13 @@ import { AggregationsStringTermsBucket, } from '@elastic/elasticsearch/lib/api/types'; import type { DataRequestHandlerContext } from '../../../data/server'; -import { getRoutePaths } from '../../common'; -import { StackTraceID } from '../../common/profiling'; +import { fromMapToRecord, getRoutePaths } from '../../common'; +import { groupStackFrameMetadataByStackTrace, StackTraceID } from '../../common/profiling'; +import { createTopNBucketsByDate } from '../../common/topn'; import { findDownsampledIndex } from './downsampling'; import { logExecutionLatency } from './logger'; import { autoHistogramSumCountOnGroupByField, newProjectTimeQuery } from './mappings'; +import { mgetExecutables, mgetStackFrames, mgetStackTraces } from './stacktrace'; export async function topNElasticSearchQuery( client: ElasticsearchClient, @@ -45,57 +47,71 @@ export async function topNElasticSearchQuery( logger, 'query to fetch events from ' + eventsIndex.name, async () => { - return await client.search({ - index: eventsIndex.name, - body: { + return await client.search( + { + index: eventsIndex.name, + size: 0, query: filter, aggs: { histogram: autoHistogramSumCountOnGroupByField(searchField, topNItems), }, }, - }); - } - ); - - let totalCount = 0; - const stackTraceEvents = new Set(); - - (resEvents.body.aggregations?.histogram as AggregationsHistogramAggregate)?.buckets?.forEach( - (timeInterval: AggregationsHistogramBucket) => { - totalCount += timeInterval.doc_count; - timeInterval.group_by.buckets.forEach((stackTraceItem: AggregationsStringTermsBucket) => { - stackTraceEvents.add(stackTraceItem.key); - }); + { + // Adrien and Dario found out this is a work-around for some bug in 8.1. + // It reduces the query time by avoiding unneeded searches. + querystring: { + pre_filter_shard_size: 1, + }, + } + ); } ); - logger.info('events total count: ' + totalCount); - logger.info('unique stacktraces: ' + stackTraceEvents.size); + const histogram = resEvents.body.aggregations?.histogram as AggregationsHistogramAggregate; + const topN = createTopNBucketsByDate(histogram); if (searchField !== 'StackTraceID') { return response.ok({ - body: { - topN: resEvents.body.aggregations, - }, + body: topN, }); } - const resTraceMetadata = await logExecutionLatency( + let totalCount = 0; + const stackTraceEvents = new Map(); + + const histogramBuckets = (histogram?.buckets as AggregationsHistogramBucket[]) ?? []; + for (let i = 0; i < histogramBuckets.length; i++) { + totalCount += histogramBuckets[i].doc_count; + histogramBuckets[i].group_by.buckets.forEach( + (stackTraceItem: AggregationsStringTermsBucket) => { + stackTraceEvents.set(stackTraceItem.key, stackTraceItem.count.value); + } + ); + } + + logger.info('events total count: ' + totalCount); + logger.info('unique stacktraces: ' + stackTraceEvents.size); + + // profiling-stacktraces is configured with 16 shards + const { stackTraces, stackFrameDocIDs, executableDocIDs } = await mgetStackTraces( logger, - 'query for ' + stackTraceEvents.size + ' stacktraces', - async () => { - return await client.mget({ - index: 'profiling-stacktraces', - body: { ids: [...stackTraceEvents] }, - }); - } + client, + stackTraceEvents ); - return response.ok({ - body: { - topN: resEvents.body.aggregations, - traceMetadata: resTraceMetadata.body.docs, - }, + return Promise.all([ + mgetStackFrames(logger, client, stackFrameDocIDs), + mgetExecutables(logger, client, executableDocIDs), + ]).then(([stackFrames, executables]) => { + const metadata = fromMapToRecord( + groupStackFrameMetadataByStackTrace(stackTraces, stackFrames, executables) + ); + return response.ok({ + body: { + ...topN, + Metadata: metadata, + }, + }); }); }