[SLO Form] Use saved Data view id , handle runtime mappings (#176662)

## Summary Fixes #173771 Use saved data view id instead of index pattern where it's available. Inject runtime mappings from the dataview into transform. - [ ] Go to Discover and add a runtime field to the data view (this is only available in Discover) - [ ] Make sure filtering works based on the data view We are not supporting "scripted fields" from the Index Management DataView editor. --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Jun 7, 2024 · f3fdb0f · f3fdb0f
1 parent 217e29d
commit f3fdb0f
Show file tree

Hide file tree

Showing 53 changed files with 1,017 additions and 482 deletions.
diff --git a/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts b/x-pack/packages/kbn-slo-schema/src/schema/indicators.ts
@@ -25,9 +25,9 @@ const filtersSchema = t.array(
       isMultiIndex: t.boolean,
       type: t.string,
       key: t.string,
+      field: t.string,
       params: t.any,
       value: t.string,
-      field: t.string,
     }),
     query: t.record(t.string, t.any),
   })
@@ -54,6 +54,7 @@ const apmTransactionDurationIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -71,6 +72,7 @@ const apmTransactionErrorRateIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -87,6 +89,7 @@ const kqlCustomIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -164,6 +167,7 @@ const timesliceMetricIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -205,6 +209,7 @@ const metricCustomIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -250,6 +255,7 @@ const histogramIndicatorSchema = t.type({
     }),
     t.partial({
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });
@@ -270,6 +276,7 @@ const syntheticsAvailabilityIndicatorSchema = t.type({
       tags: t.array(syntheticsParamSchema),
       projects: t.array(syntheticsParamSchema),
       filter: querySchema,
+      dataViewId: t.string,
     }),
   ]),
 });

diff --git a/x-pack/plugins/observability_solution/observability/server/plugin.ts b/x-pack/plugins/observability_solution/observability/server/plugin.ts
@@ -36,6 +36,7 @@ import { RuleRegistryPluginSetupContract } from '@kbn/rule-registry-plugin/serve
 import { SharePluginSetup } from '@kbn/share-plugin/server';
 import { SpacesPluginSetup, SpacesPluginStart } from '@kbn/spaces-plugin/server';
 import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server';
+import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import { ObservabilityConfig } from '.';
 import { casesFeatureId, observabilityFeatureId } from '../common';
 import {
@@ -71,6 +72,7 @@ interface PluginSetup {
 interface PluginStart {
   alerting: PluginStartContract;
   spaces?: SpacesPluginStart;
+  dataViews: DataViewsServerPluginStart;
 }
 
 const o11yRuleTypes = [
@@ -294,6 +296,7 @@ export class ObservabilityPlugin implements Plugin<ObservabilityPluginSetup> {
             ...plugins,
             core,
           },
+          dataViews: pluginStart.dataViews,
           spaces: pluginStart.spaces,
           ruleDataService,
           assistant: {

diff --git a/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts b/x-pack/plugins/observability_solution/observability/server/routes/register_routes.ts
@@ -17,6 +17,7 @@ import {
 import { SpacesPluginStart } from '@kbn/spaces-plugin/server';
 import axios from 'axios';
 import * as t from 'io-ts';
+import { DataViewsServerPluginStart } from '@kbn/data-views-plugin/server';
 import { ObservabilityConfig } from '..';
 import { getHTTPResponseCode, ObservabilityError } from '../errors';
 import { AlertDetailsContextualInsightsService } from '../services';
@@ -35,6 +36,7 @@ export interface RegisterRoutesDependencies {
   pluginsSetup: {
     core: CoreSetup;
   };
+  dataViews: DataViewsServerPluginStart;
   spaces?: SpacesPluginStart;
   ruleDataService: RuleDataPluginService;
   assistant: {

diff --git a/...ity_solution/slo/docs/openapi/slo/components/schemas/indicator_properties_custom_kql.yaml b/...ity_solution/slo/docs/openapi/slo/components/schemas/indicator_properties_custom_kql.yaml
@@ -19,6 +19,11 @@ properties:
         description: The index or index pattern to use
         type: string
         example: my-service-*
+      dataViewId:
+        description: The kibana data view id to use, primarily used to include data view runtime mappings.
+                     Make sure to save SLO again if you add/update run time fields to the data view and if those fields are being used in slo queries.
+        type: string
+        example: 03b80ab3-003d-498b-881c-3beedbaf1162
       filter:
         description: the KQL query to filter the documents with.
         $ref: "kql_with_filters.yaml"

diff --git a/..._solution/slo/docs/openapi/slo/components/schemas/indicator_properties_custom_metric.yaml b/..._solution/slo/docs/openapi/slo/components/schemas/indicator_properties_custom_metric.yaml
@@ -19,6 +19,11 @@ properties:
         description: The index or index pattern to use
         type: string
         example: my-service-*
+      dataViewId:
+        description: The kibana data view id to use, primarily used to include data view runtime mappings.
+          Make sure to save SLO again if you add/update run time fields to the data view and if those fields are being used in slo queries.
+        type: string
+        example: 03b80ab3-003d-498b-881c-3beedbaf1162
       filter:
         description: the KQL query to filter the documents with.
         type: string

diff --git a/...lity_solution/slo/docs/openapi/slo/components/schemas/indicator_properties_histogram.yaml b/...lity_solution/slo/docs/openapi/slo/components/schemas/indicator_properties_histogram.yaml
@@ -19,6 +19,11 @@ properties:
         description: The index or index pattern to use
         type: string
         example: my-service-*
+      dataViewId:
+        description: The kibana data view id to use, primarily used to include data view runtime mappings.
+          Make sure to save SLO again if you add/update run time fields to the data view and if those fields are being used in slo queries.
+        type: string
+        example: 03b80ab3-003d-498b-881c-3beedbaf1162
       filter:
         description: the KQL query to filter the documents with.
         type: string

diff --git a/...lution/slo/docs/openapi/slo/components/schemas/indicator_properties_timeslice_metric.yaml b/...lution/slo/docs/openapi/slo/components/schemas/indicator_properties_timeslice_metric.yaml
@@ -18,6 +18,11 @@ properties:
         description: The index or index pattern to use
         type: string
         example: my-service-*
+      dataViewId:
+        description: The kibana data view id to use, primarily used to include data view runtime mappings.
+          Make sure to save SLO again if you add/update run time fields to the data view and if those fields are being used in slo queries.
+        type: string
+        example: 03b80ab3-003d-498b-881c-3beedbaf1162
       filter:
         description: the KQL query to filter the documents with.
         type: string

diff --git a/x-pack/plugins/observability_solution/slo/kibana.jsonc b/x-pack/plugins/observability_solution/slo/kibana.jsonc
@@ -22,6 +22,7 @@
       "dataViews",
       "lens",
       "dataViewEditor",
+      "dataViewFieldEditor",
       "fieldFormats",
       "observability",
       "observabilityShared",

diff --git a/x-pack/plugins/observability_solution/slo/public/hooks/use_create_data_view.ts b/x-pack/plugins/observability_solution/slo/public/hooks/use_create_data_view.ts
@@ -5,39 +5,36 @@
  * 2.0.
  */
 
-import { useEffect, useState } from 'react';
-import { DataView } from '@kbn/data-views-plugin/common';
+import { useFetcher } from '@kbn/observability-shared-plugin/public';
 import { useKibana } from '../utils/kibana_react';
 
 interface UseCreateDataViewProps {
   indexPatternString?: string;
+  dataViewId?: string;
 }
 
-export function useCreateDataView({ indexPatternString }: UseCreateDataViewProps) {
+export function useCreateDataView({ indexPatternString, dataViewId }: UseCreateDataViewProps) {
   const { dataViews } = useKibana().services;
 
-  const [stateDataView, setStateDataView] = useState<DataView | undefined>();
-  const [isLoading, setIsLoading] = useState<boolean>(false);
-
-  useEffect(() => {
-    const createDataView = () =>
-      dataViews.create({
+  const { data: dataView, loading } = useFetcher(async () => {
+    if (dataViewId) {
+      try {
+        return await dataViews.get(dataViewId);
+      } catch (e) {
+        return dataViews.create({
+          id: `${indexPatternString}-id`,
+          title: indexPatternString,
+          allowNoIndex: true,
+        });
+      }
+    } else if (indexPatternString) {
+      return dataViews.create({
         id: `${indexPatternString}-id`,
         title: indexPatternString,
         allowNoIndex: true,
       });
-
-    if (indexPatternString) {
-      setIsLoading(true);
-      createDataView()
-        .then((value) => {
-          setStateDataView(value);
-        })
-        .finally(() => {
-          setIsLoading(false);
-        });
     }
-  }, [indexPatternString, dataViews]);
+  }, [dataViewId, dataViews, indexPatternString]);
 
-  return { dataView: stateDataView, loading: isLoading };
+  return { dataView, loading: Boolean(loading) };
 }
diff --git a/x-pack/plugins/observability_solution/slo/public/pages/slo_details/slo_details.test.tsx b/x-pack/plugins/observability_solution/slo/public/pages/slo_details/slo_details.test.tsx
@@ -32,6 +32,7 @@ import { useKibana } from '../../utils/kibana_react';
 import { render } from '../../utils/test_helper';
 import { SloDetailsPage } from './slo_details';
 import { TagsList, HeaderMenuPortal } from '@kbn/observability-shared-plugin/public';
+import { useCreateDataView } from '../../hooks/use_create_data_view';
 
 jest.mock('react-router-dom', () => ({
   ...jest.requireActual('react-router-dom'),
@@ -46,6 +47,7 @@ jest.mock('../../hooks/use_fetch_active_alerts');
 jest.mock('../../hooks/use_fetch_slo_details');
 jest.mock('../../hooks/use_fetch_historical_summary');
 jest.mock('../../hooks/use_delete_slo');
+jest.mock('../../hooks/use_create_data_view');
 jest.mock('../../hooks/use_delete_slo_instance');
 
 const useKibanaMock = useKibana as jest.Mock;
@@ -56,6 +58,7 @@ const useFetchActiveAlertsMock = useFetchActiveAlerts as jest.Mock;
 const useFetchSloDetailsMock = useFetchSloDetails as jest.Mock;
 const useFetchHistoricalSummaryMock = useFetchHistoricalSummary as jest.Mock;
 const useDeleteSloMock = useDeleteSlo as jest.Mock;
+const useCreateDataViewsMock = useCreateDataView as jest.Mock;
 const useDeleteSloInstanceMock = useDeleteSloInstance as jest.Mock;
 const TagsListMock = TagsList as jest.Mock;
 TagsListMock.mockReturnValue(<div>Tags list</div>);
@@ -132,6 +135,9 @@ describe('SLO Details Page', () => {
     jest.clearAllMocks();
     mockKibana();
     useCapabilitiesMock.mockReturnValue({ hasWriteCapabilities: true, hasReadCapabilities: true });
+    useCreateDataViewsMock.mockReturnValue({
+      dataView: { getName: () => 'dataview', getIndexPattern: () => '.dataview-index' },
+    });
     useFetchHistoricalSummaryMock.mockReturnValue({
       isLoading: false,
       data: historicalSummaryData,

diff --git a/...ublic/pages/slo_edit/components/apm_availability/apm_availability_indicator_type_form.tsx b/...ublic/pages/slo_edit/components/apm_availability/apm_availability_indicator_type_form.tsx
@@ -10,6 +10,7 @@ import { APMTransactionErrorRateIndicator } from '@kbn/slo-schema';
 import { i18n } from '@kbn/i18n';
 import React, { useEffect } from 'react';
 import { useFormContext } from 'react-hook-form';
+import { DATA_VIEW_FIELD } from '../custom_common/index_selection';
 import { useCreateDataView } from '../../../../hooks/use_create_data_view';
 import { GroupByField } from '../common/group_by_field';
 import { useFetchApmIndex } from '../../../../hooks/use_fetch_apm_indices';
@@ -23,6 +24,7 @@ import { getGroupByCardinalityFilters } from '../apm_common/get_group_by_cardina
 export function ApmAvailabilityIndicatorTypeForm() {
   const { watch, setValue } = useFormContext<CreateSLOForm<APMTransactionErrorRateIndicator>>();
   const { data: apmIndex } = useFetchApmIndex();
+  const dataViewId = watch(DATA_VIEW_FIELD);
 
   const [
     serviceName = '',
@@ -53,6 +55,7 @@ export function ApmAvailabilityIndicatorTypeForm() {
 
   const { dataView, loading: isIndexFieldsLoading } = useCreateDataView({
     indexPatternString: apmIndex,
+    dataViewId,
   });
 
   return (
@@ -129,7 +132,7 @@ export function ApmAvailabilityIndicatorTypeForm() {
         <EuiFlexItem>
           <QueryBuilder
             dataTestSubj="apmLatencyFilterInput"
-            indexPatternString={watch('indicator.params.index')}
+            dataView={dataView}
             label={i18n.translate('xpack.slo.sloEdit.apmLatency.filter', {
               defaultMessage: 'Query filter',
             })}

diff --git a/...tion/slo/public/pages/slo_edit/components/apm_latency/apm_latency_indicator_type_form.tsx b/...tion/slo/public/pages/slo_edit/components/apm_latency/apm_latency_indicator_type_form.tsx
@@ -10,6 +10,7 @@ import { APMTransactionDurationIndicator } from '@kbn/slo-schema';
 import { i18n } from '@kbn/i18n';
 import React, { useEffect } from 'react';
 import { Controller, useFormContext } from 'react-hook-form';
+import { DATA_VIEW_FIELD } from '../custom_common/index_selection';
 import { GroupByField } from '../common/group_by_field';
 import { useCreateDataView } from '../../../../hooks/use_create_data_view';
 import { useFetchApmIndex } from '../../../../hooks/use_fetch_apm_indices';
@@ -52,8 +53,11 @@ export function ApmLatencyIndicatorTypeForm() {
     }
   }, [setValue, apmIndex]);
 
+  const dataViewId = watch(DATA_VIEW_FIELD);
+
   const { dataView, loading: isIndexFieldsLoading } = useCreateDataView({
     indexPatternString: apmIndex,
+    dataViewId,
   });
 
   return (
@@ -164,7 +168,7 @@ export function ApmLatencyIndicatorTypeForm() {
         <EuiFlexItem>
           <QueryBuilder
             dataTestSubj="apmLatencyFilterInput"
-            indexPatternString={watch('indicator.params.index')}
+            dataView={dataView}
             label={i18n.translate('xpack.slo.sloEdit.apmLatency.filter', {
               defaultMessage: 'Query filter',
             })}

diff --git a/...gins/observability_solution/slo/public/pages/slo_edit/components/common/query_builder.tsx b/...gins/observability_solution/slo/public/pages/slo_edit/components/common/query_builder.tsx
@@ -9,14 +9,15 @@ import { i18n } from '@kbn/i18n';
 import { EuiButtonIcon, EuiFlexGroup, EuiFlexItem } from '@elastic/eui';
 import React, { ReactNode, useState } from 'react';
 import { FieldPath } from 'react-hook-form';
+import { DataView } from '@kbn/data-views-plugin/common';
+import { RunTimeFieldUsed } from './runtime_field_used';
 import { QuerySearchBar } from './query_search_bar';
 import { QueryDocumentsFlyout } from './query_documents_flyout';
-import { useCreateDataView } from '../../../../hooks/use_create_data_view';
 import { CreateSLOForm } from '../../types';
 
 export interface SearchBarProps {
   dataTestSubj: string;
-  indexPatternString: string | undefined;
+  dataView?: DataView;
   label: string;
   name: FieldPath<CreateSLOForm>;
   placeholder: string;
@@ -25,10 +26,7 @@ export interface SearchBarProps {
 }
 
 export function QueryBuilder(props: SearchBarProps) {
-  const { indexPatternString, name } = props;
-  const { dataView } = useCreateDataView({
-    indexPatternString,
-  });
+  const { dataView, name } = props;
 
   const [isFlyoutOpen, setIsFlyoutOpen] = useState(false);
   const [range, setRange] = useState({ from: 'now-15m', to: 'now' });
@@ -67,6 +65,7 @@ export function QueryBuilder(props: SearchBarProps) {
           searchBarProps={props}
         />
       )}
+      <RunTimeFieldUsed dataView={dataView} name={name} />
     </>
   );
 }
diff --git a/...s/observability_solution/slo/public/pages/slo_edit/components/common/query_search_bar.tsx b/...s/observability_solution/slo/public/pages/slo_edit/components/common/query_search_bar.tsx
@@ -12,7 +12,6 @@ import { kqlQuerySchema, kqlWithFiltersSchema } from '@kbn/slo-schema';
 import React, { memo } from 'react';
 import styled from 'styled-components';
 import { observabilityAppId } from '@kbn/observability-shared-plugin/common';
-import { useCreateDataView } from '../../../../hooks/use_create_data_view';
 import { SearchBarProps } from './query_builder';
 import { useKibana } from '../../../../utils/kibana_react';
 import { CreateSLOForm } from '../../types';
@@ -23,7 +22,7 @@ export const QuerySearchBar = memo(
     isFlyoutOpen,
     name,
     label,
-    indexPatternString,
+    dataView,
     required,
     tooltip,
     dataTestSubj,
@@ -36,9 +35,6 @@ export const QuerySearchBar = memo(
     setRange: (range: TimeRange) => void;
   }) => {
     const { SearchBar } = useKibana().services.unifiedSearch.ui;
-    const { dataView } = useCreateDataView({
-      indexPatternString,
-    });
 
     const { control } = useFormContext<CreateSLOForm>();