[8.15] [ML] AIOps Fixing runtime mappings in pattern analysis (#188530)

x-pack/packages/ml/aiops_log_pattern_analysis/create_category_request.ts

@@ -9,6 +9,8 @@ import type {
   QueryDslQueryContainer,
   AggregationsCustomCategorizeTextAnalyzer,
 } from '@elastic/elasticsearch/lib/api/types';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+import { isPopulatedObject } from '@kbn/ml-is-populated-object/src/is_populated_object';
 
 import type { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
 
@@ -29,6 +31,7 @@ export function createCategoryRequest(
   timeField: string,
   timeRange: { from: number; to: number } | undefined,
   queryIn: QueryDslQueryContainer,
+  runtimeMappings: MappingRuntimeFields | undefined,
   wrap: ReturnType<typeof createRandomSamplerWrapper>['wrap'],
   intervalMs?: number,
   additionalFilter?: CategorizationAdditionalFilter,
@@ -113,6 +116,7 @@ export function createCategoryRequest(
       body: {
         query,
         aggs: wrap(aggs),
+        ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}),
       },
     },
   };

x-pack/packages/ml/aiops_log_pattern_analysis/tsconfig.json

@@ -23,5 +23,6 @@
     "@kbn/es-query",
     "@kbn/saved-search-plugin",
     "@kbn/data-views-plugin",
+    "@kbn/ml-is-populated-object",
   ]
 }

x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_categories.ts

@@ -75,6 +75,7 @@ export const getCategoryRequest = (
     timeFieldName,
     undefined,
     query,
+    undefined,
     wrap,
     undefined,
     undefined,

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/log_categorization_for_embeddable.tsx

@@ -247,30 +247,41 @@ export const LogCategorizationEmbeddable: FC<LogCategorizationEmbeddableProps> =
       from: earliest,
       to: latest,
     };
+    const runtimeMappings = dataView.getRuntimeMappings();
 
     try {
       const timeRange = await getMinimumTimeRange(
         index,
         timeField,
         additionalFilter,
         minimumTimeRangeOption,
-        searchQuery
+        searchQuery,
+        runtimeMappings
       );
 
       if (mounted.current !== true) {
         return;
       }
 
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField.name, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
+        runValidateFieldRequest(
+          index,
+          selectedField.name,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
         runCategorizeRequest(
           index,
           selectedField.name,
           timeField,
           { to: timeRange.to, from: timeRange.from },
           searchQuery,
+          runtimeMappings,
           intervalMs,
           timeRange.useSubAgg ? additionalFilter : undefined
         ),

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_embeddable/use_minimum_time_range.ts

@@ -12,6 +12,7 @@ import type { HttpFetchOptions } from '@kbn/core/public';
 import { getTimeFieldRange } from '@kbn/ml-date-picker';
 import moment from 'moment';
 import { useStorage } from '@kbn/ml-local-storage';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { useAiopsAppContext } from '../../../hooks/use_aiops_app_context';
 import type { MinimumTimeRangeOption } from './minimum_time_range';
 import { MINIMUM_TIME_RANGE } from './minimum_time_range';
@@ -29,6 +30,7 @@ export function useMinimumTimeRange() {
       timeRange: { from: number; to: number },
       minimumTimeRangeOption: MinimumTimeRangeOption,
       queryIn: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       headers?: HttpFetchOptions['headers']
     ) => {
       const minimumTimeRange = MINIMUM_TIME_RANGE[minimumTimeRangeOption];
@@ -47,6 +49,7 @@ export function useMinimumTimeRange() {
         index,
         timeFieldName: timeField,
         query: queryIn,
+        runtimeMappings,
         path: '/internal/file_upload/time_field_range',
         signal: abortController.current.signal,
       });

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx

@@ -189,17 +189,28 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
       to: latest,
     };
 
+    const runtimeMappings = dataView.getRuntimeMappings();
+
     try {
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField.name, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
+        runValidateFieldRequest(
+          index,
+          selectedField.name,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
         runCategorizeRequest(
           index,
           selectedField.name,
           timeField,
           timeRange,
           searchQuery,
+          runtimeMappings,
           intervalMs,
           additionalFilter
         ),

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx

@@ -217,13 +217,31 @@ export const LogCategorizationPage: FC<LogCategorizationPageProps> = ({ embeddin
       to: latest,
     };
 
+    const runtimeMappings = dataView.getRuntimeMappings();
+
     try {
       const [validationResult, categorizationResult] = await Promise.all([
-        runValidateFieldRequest(index, selectedField, timeField, timeRange, searchQuery, {
-          [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
-        }),
-
-        runCategorizeRequest(index, selectedField, timeField, timeRange, searchQuery, intervalMs),
+        runValidateFieldRequest(
+          index,
+          selectedField,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          {
+            [AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN]: embeddingOrigin,
+          }
+        ),
+
+        runCategorizeRequest(
+          index,
+          selectedField,
+          timeField,
+          timeRange,
+          searchQuery,
+          runtimeMappings,
+          intervalMs
+        ),
       ]);
 
       setFieldValidationResult(validationResult);

x-pack/plugins/aiops/public/components/log_categorization/use_categorize_request.ts

@@ -19,6 +19,7 @@ import {
 import { processCategoryResults } from '@kbn/aiops-log-pattern-analysis/process_category_results';
 import type { CatResponse } from '@kbn/aiops-log-pattern-analysis/types';
 
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import {
   type AiOpsKey,
@@ -69,6 +70,7 @@ export function useCategorizeRequest() {
       timeField: string,
       timeRange: { from: number; to: number },
       query: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       intervalMs?: number,
       additionalFilter?: CategorizationAdditionalFilter
     ): Promise<ReturnType<typeof processCategoryResults>> => {
@@ -83,6 +85,7 @@ export function useCategorizeRequest() {
               timeField,
               timeRange,
               query,
+              runtimeMappings,
               wrap,
               intervalMs,
               additionalFilter,

x-pack/plugins/aiops/public/components/log_categorization/use_validate_category_field.ts

@@ -8,6 +8,7 @@
 import { useRef, useCallback } from 'react';
 
 import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import type { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import type { HttpFetchOptions } from '@kbn/core/public';
@@ -28,6 +29,7 @@ export function useValidateFieldRequest() {
       timeField: string,
       timeRange: { from: number; to: number },
       queryIn: QueryDslQueryContainer,
+      runtimeMappings: MappingRuntimeFields | undefined,
       headers?: HttpFetchOptions['headers']
     ) => {
       const query = createCategorizeQuery(queryIn, timeField, timeRange);
@@ -42,10 +44,7 @@ export function useValidateFieldRequest() {
             timeField,
             start: timeRange.from,
             end: timeRange.to,
-            // only text fields are supported in pattern analysis,
-            // and it is not possible to create a text runtime field
-            // so runtimeMappings are not needed
-            runtimeMappings: undefined,
+            runtimeMappings,
             indicesOptions: undefined,
             includeExamples: false,
           }),