Adding size to serverless

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_stream_details/index.ts

@@ -20,6 +20,7 @@ import { DataStreamDetails, DataStreamSettings } from '../../../../common/api_ty
 import { createDatasetQualityESClient } from '../../../utils';
 import { dataStreamService, datasetQualityPrivileges } from '../../../services';
 import { getDataStreams } from '../get_data_streams';
+import { getDataStreamsMeteringStats } from '../get_data_streams_metering_stats';
 
 export async function getDataStreamSettings({
   esClient,
@@ -80,12 +81,13 @@ export async function getDataStreamDetails({
       end
     );
 
-    const whenSizeStatsNotAvailable = NaN; // This will indicate size cannot be calculated
-    const avgDocSizeInBytes = isServerless
-      ? whenSizeStatsNotAvailable
-      : hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
-      ? await getAvgDocSizeInBytes(esClient, dataStream)
-      : 0;
+    const avgDocSizeInBytes =
+      hasAccessToDataStream && dataStreamSummaryStats.docsCount > 0
+        ? isServerless
+          ? await getMeteringAvgDocSizeInBytes(esClient, dataStream)
+          : await getAvgDocSizeInBytes(esClient, dataStream)
+        : 0;
+
     const sizeBytes = Math.ceil(avgDocSizeInBytes * dataStreamSummaryStats.docsCount);
 
     return {
@@ -172,6 +174,18 @@ async function getDataStreamSummaryStats(
   };
 }
 
+async function getMeteringAvgDocSizeInBytes(esClient: ElasticsearchClient, index: string) {
+  const meteringStats = await getDataStreamsMeteringStats({
+    esClient,
+    dataStreams: [index],
+  });
+
+  const docCount = meteringStats[index].totalDocs ?? 0;
+  const sizeInBytes = meteringStats[index].sizeBytes ?? 0;
+
+  return docCount ? sizeInBytes / docCount : 0;
+}
+
 async function getAvgDocSizeInBytes(esClient: ElasticsearchClient, index: string) {
   const indexStats = await esClient.indices.stats({ index });
   const docCount = indexStats._all.total?.docs?.count ?? 0;

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/get_data_streams_metering_stats/index.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ElasticsearchClient } from '@kbn/core/server';
+
+export interface MeteringStatsResponse {
+  datastreams: Array<{
+    name: string;
+    num_docs: number;
+    size_in_bytes: number;
+  }>;
+}
+
+export async function getDataStreamsMeteringStats({
+  esClient,
+  dataStreams,
+}: {
+  esClient: ElasticsearchClient;
+  dataStreams: string[];
+}): Promise<Record<string, { size?: string; sizeBytes: number; totalDocs: number }>> {
+  if (!dataStreams.length) {
+    return {};
+  }
+
+  const { datastreams: dataStreamsStats } = await esClient.transport.request<MeteringStatsResponse>(
+    {
+      method: 'GET',
+      path: `/_metering/stats/` + dataStreams.join(','),
+    }
+  );
+
+  return dataStreamsStats.reduce(
+    (acc, dataStream) => ({
+      ...acc,
+      [dataStream.name]: {
+        sizeBytes: dataStream.size_in_bytes,
+        totalDocs: dataStream.num_docs,
+      },
+    }),
+    {}
+  );
+}

x-pack/plugins/observability_solution/dataset_quality/server/routes/data_streams/routes.ts

@@ -26,6 +26,7 @@ import { getDegradedDocsPaginated } from './get_degraded_docs';
 import { getNonAggregatableDataStreams } from './get_non_aggregatable_data_streams';
 import { getDegradedFields } from './get_degraded_fields';
 import { getDegradedFieldValues } from './get_degraded_field_values';
+import { getDataStreamsMeteringStats } from './get_data_streams_metering_stats';
 
 const statsRoute = createDatasetQualityServerRoute({
   endpoint: 'GET /internal/dataset_quality/data_streams/stats',
@@ -50,6 +51,7 @@ const statsRoute = createDatasetQualityServerRoute({
 
     // Query datastreams as the current user as the Kibana internal user may not have all the required permissions
     const esClient = coreContext.elasticsearch.client.asCurrentUser;
+    const esClientAsSecondaryAuthUser = coreContext.elasticsearch.client.asSecondaryAuthUser;
 
     const { dataStreams, datasetUserPrivileges } = await getDataStreams({
       esClient,
@@ -62,7 +64,10 @@ const statsRoute = createDatasetQualityServerRoute({
     });
 
     const dataStreamsStats = isServerless
-      ? {}
+      ? await getDataStreamsMeteringStats({
+          esClient: esClientAsSecondaryAuthUser,
+          dataStreams: privilegedDataStreams.map((stream) => stream.name),
+        })
       : await getDataStreamsStats({
           esClient,
           dataStreams: privilegedDataStreams.map((stream) => stream.name),

x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_details.ts

@@ -35,7 +35,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
   const retry = getService('retry');
   const browser = getService('browser');
   const to = '2024-01-01T12:00:00.000Z';
-  const excludeKeysFromServerless = ['size']; // https://github.com/elastic/kibana/issues/178954
 
   const apacheAccessDatasetName = 'apache.access';
   const apacheAccessDataStreamName = `logs-${apacheAccessDatasetName}-${productionNamespace}`;
@@ -172,12 +171,13 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
           dataStream: apacheAccessDataStreamName,
         });
 
-        const { docsCountTotal, degradedDocs, services, hosts } =
-          await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis(excludeKeysFromServerless);
+        const { docsCountTotal, degradedDocs, services, hosts, size } =
+          await PageObjects.datasetQuality.parseOverviewSummaryPanelKpis();
         expect(parseInt(docsCountTotal, 10)).to.be(226);
         expect(parseInt(degradedDocs, 10)).to.be(1);
         expect(parseInt(services, 10)).to.be(3);
         expect(parseInt(hosts, 10)).to.be(52);
+        expect(parseInt(size, 10)).to.be.greaterThan(0);
       });
     });
 

x-pack/test_serverless/functional/test_suites/observability/dataset_quality/dataset_quality_table.ts

@@ -112,6 +112,17 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) {
       expect(degradedDocsColCellTexts).to.eql(['0%', '0%', '0%', '100%']);
     });
 
+    it('shows the value in the size column', async () => {
+      const cols = await PageObjects.datasetQuality.parseDatasetTable();
+
+      const sizeColCellTexts = await cols.Size.getCellTexts();
+      const sizeGreaterThanZero = sizeColCellTexts[3];
+      const sizeEqualToZero = sizeColCellTexts[2];
+
+      expect(sizeGreaterThanZero).to.not.eql('0.0 KB');
+      expect(sizeEqualToZero).to.eql('0.0 B');
+    });
+
     it('shows dataset from integration', async () => {
       const cols = await PageObjects.datasetQuality.parseDatasetTable();
       const datasetNameCol = cols['Data Set Name'];