[ML] AIOps: Move log pattern analysis fetch code to common to be av…

…ailable via `server` (elastic#167465)

Follow up to elastic#167237.
Part of elastic#167467.

We plan to reuse some of the queries log pattern analysis does to use
via log rate analysis too. Log pattern analysis mostly does queries from
the client side, late rate analysis has its own API endpoint and does ES
queries via Kibana server. In preparation for the use via log rate
analysis, this moves the code we need to have available server side for
log rate analysis to the `common` area of the plugin so it can be used
both on server/client.

x-pack/plugins/aiops/common/api/log_categorization/create_categorize_query.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { cloneDeep } from 'lodash';
+
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+
+export function createCategorizeQuery(
+  queryIn: QueryDslQueryContainer,
+  timeField: string,
+  from: number | undefined,
+  to: number | undefined
+) {
+  const query = cloneDeep(queryIn);
+
+  if (query.bool === undefined) {
+    query.bool = {};
+  }
+  if (query.bool.must === undefined) {
+    query.bool.must = [];
+    if (query.match_all !== undefined) {
+      query.bool.must.push({ match_all: query.match_all });
+      delete query.match_all;
+    }
+  }
+  if (query.multi_match !== undefined) {
+    query.bool.should = {
+      multi_match: query.multi_match,
+    };
+    delete query.multi_match;
+  }
+
+  (query.bool.must as QueryDslQueryContainer[]).push({
+    range: {
+      [timeField]: {
+        gte: from,
+        lte: to,
+        format: 'epoch_millis',
+      },
+    },
+  });
+
+  return query;
+}

x-pack/plugins/aiops/common/api/log_categorization/create_category_request.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+
+import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
+
+import { createCategorizeQuery } from './create_categorize_query';
+
+const CATEGORY_LIMIT = 1000;
+const EXAMPLE_LIMIT = 1;
+
+export function createCategoryRequest(
+  index: string,
+  field: string,
+  timeField: string,
+  from: number | undefined,
+  to: number | undefined,
+  queryIn: QueryDslQueryContainer,
+  wrap: ReturnType<typeof createRandomSamplerWrapper>['wrap'],
+  intervalMs?: number
+) {
+  const query = createCategorizeQuery(queryIn, timeField, from, to);
+  const aggs = {
+    categories: {
+      categorize_text: {
+        field,
+        size: CATEGORY_LIMIT,
+      },
+      aggs: {
+        hit: {
+          top_hits: {
+            size: EXAMPLE_LIMIT,
+            sort: [timeField],
+            _source: field,
+          },
+        },
+        ...(intervalMs
+          ? {
+              sparkline: {
+                date_histogram: {
+                  field: timeField,
+                  fixed_interval: `${intervalMs}ms`,
+                },
+              },
+            }
+          : {}),
+      },
+    },
+  };
+
+  return {
+    params: {
+      index,
+      size: 0,
+      body: {
+        query,
+        aggs: wrap(aggs),
+      },
+    },
+  };
+}

x-pack/plugins/aiops/common/api/log_categorization/get_category_query.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { Category } from './types';
+
+export const QUERY_MODE = {
+  INCLUDE: 'should',
+  EXCLUDE: 'must_not',
+} as const;
+export type QueryMode = typeof QUERY_MODE[keyof typeof QUERY_MODE];
+
+export const getCategoryQuery = (
+  field: string,
+  categories: Category[],
+  mode: QueryMode = QUERY_MODE.INCLUDE
+) => ({
+  bool: {
+    [mode]: categories.map(({ key: query }) => ({
+      match: {
+        [field]: {
+          auto_generate_synonyms_phrase_query: false,
+          fuzziness: 0,
+          operator: 'and',
+          query,
+        },
+      },
+    })),
+  },
+});

x-pack/plugins/aiops/common/api/log_categorization/process_category_results.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { get } from 'lodash';
+
+import { estypes } from '@elastic/elasticsearch';
+
+import { createRandomSamplerWrapper } from '@kbn/ml-random-sampler-utils';
+
+import type { Category, CategoriesAgg, CatResponse, SparkLinesPerCategory } from './types';
+
+export function processCategoryResults(
+  result: CatResponse,
+  field: string,
+  unwrap: ReturnType<typeof createRandomSamplerWrapper>['unwrap']
+) {
+  const sparkLinesPerCategory: SparkLinesPerCategory = {};
+  const { aggregations } = result.rawResponse;
+  if (aggregations === undefined) {
+    throw new Error('processCategoryResults failed, did not return aggregations.');
+  }
+  const {
+    categories: { buckets },
+  } = unwrap(
+    aggregations as unknown as Record<string, estypes.AggregationsAggregate>
+  ) as CategoriesAgg;
+
+  const categories: Category[] = buckets.map((b) => {
+    sparkLinesPerCategory[b.key] =
+      b.sparkline === undefined
+        ? {}
+        : b.sparkline.buckets.reduce<Record<number, number>>((acc2, cur2) => {
+            acc2[cur2.key] = cur2.doc_count;
+            return acc2;
+          }, {});
+
+    return {
+      key: b.key,
+      count: b.doc_count,
+      examples: b.hit.hits.hits.map((h) => get(h._source, field)),
+    };
+  });
+  return {
+    categories,
+    sparkLinesPerCategory,
+  };
+}

x-pack/plugins/aiops/common/api/log_categorization/types.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+
+export interface Category {
+  key: string;
+  count: number;
+  examples: string[];
+  sparkline?: Array<{ doc_count: number; key: number; key_as_string: string }>;
+}
+
+export interface CategoriesAgg {
+  categories: {
+    buckets: Array<{
+      key: string;
+      doc_count: number;
+      hit: { hits: { hits: Array<{ _source: { message: string } }> } };
+      sparkline: {
+        buckets: Array<{ key_as_string: string; key: number; doc_count: number }>;
+      };
+    }>;
+  };
+}
+
+interface CategoriesSampleAgg {
+  sample: CategoriesAgg;
+}
+
+export interface CatResponse {
+  rawResponse: estypes.SearchResponseBody<unknown, CategoriesAgg | CategoriesSampleAgg>;
+}
+
+export type SparkLinesPerCategory = Record<string, Record<number, number>>;

x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx

@@ -6,8 +6,7 @@
  */
 
 import React, { FC, useMemo, useState } from 'react';
-import { i18n } from '@kbn/i18n';
-import type { TimefilterContract } from '@kbn/data-plugin/public';
+
 import {
   useEuiBackgroundColor,
   EuiInMemoryTable,
@@ -19,14 +18,25 @@ import {
   EuiSpacer,
 } from '@elastic/eui';
 
+import { i18n } from '@kbn/i18n';
+import type { TimefilterContract } from '@kbn/data-plugin/public';
 import { DataViewField } from '@kbn/data-views-plugin/common';
 import { Filter } from '@kbn/es-query';
 import { useTableState } from '@kbn/ml-in-memory-table';
-import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
-import { MiniHistogram } from '../../mini_histogram';
+
+import type {
+  Category,
+  SparkLinesPerCategory,
+} from '../../../../common/api/log_categorization/types';
+
 import { useEuiTheme } from '../../../hooks/use_eui_theme';
 import type { LogCategorizationAppState } from '../../../application/utils/url_state';
-import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request';
+
+import { MiniHistogram } from '../../mini_histogram';
+
+import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
+import type { EventRate } from '../use_categorize_request';
+
 import { getLabels } from './labels';
 import { TableHeader } from './table_header';
 

x-pack/plugins/aiops/public/components/log_categorization/document_count_chart.tsx

@@ -9,11 +9,15 @@ import React, { FC, useMemo } from 'react';
 
 import { i18n } from '@kbn/i18n';
 import { DocumentCountChart as DocumentCountChartRoot } from '@kbn/aiops-components';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
+import { DocumentCountStats } from '../../get_document_stats';
+
 import { TotalCountHeader } from '../document_count_content/total_count_header';
-import type { Category, SparkLinesPerCategory } from './use_categorize_request';
+
 import type { EventRate } from './use_categorize_request';
-import { DocumentCountStats } from '../../get_document_stats';
 
 interface Props {
   totalCount: number;

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx

@@ -5,10 +5,7 @@
  * 2.0.
  */
 import React, { FC, useState, useEffect, useCallback, useRef, useMemo } from 'react';
-import type { SavedSearch } from '@kbn/saved-search-plugin/public';
-import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
-import { i18n } from '@kbn/i18n';
-import { FormattedMessage } from '@kbn/i18n-react';
+
 import {
   EuiTitle,
   EuiFlyoutHeader,
@@ -18,26 +15,33 @@ import {
   useEuiTheme,
 } from '@elastic/eui';
 
+import type { SavedSearch } from '@kbn/saved-search-plugin/public';
+import type { DataView, DataViewField } from '@kbn/data-views-plugin/public';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n-react';
 import { buildEmptyFilter, Filter } from '@kbn/es-query';
-
 import { usePageUrlState } from '@kbn/ml-url-state';
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
+import {
+  type LogCategorizationPageUrlState,
+  getDefaultLogCategorizationAppState,
+} from '../../application/utils/url_state';
+import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
+import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
+
 import { useCategorizeRequest } from './use_categorize_request';
-import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request';
+import type { EventRate } from './use_categorize_request';
 import { CategoryTable } from './category_table';
-import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { InformationText } from './information_text';
-import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { SamplingMenu } from './sampling_menu';
 import { TechnicalPreviewBadge } from './technical_preview_badge';
 import { LoadingCategorization } from './loading_categorization';
 import { useValidateFieldRequest } from './use_validate_category_field';
-import {
-  type LogCategorizationPageUrlState,
-  getDefaultLogCategorizationAppState,
-} from '../../application/utils/url_state';
 import { FieldValidationCallout } from './category_validation_callout';
 
 export interface LogCategorizationPageProps {

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_page.tsx

@@ -24,9 +24,11 @@ import { Filter, Query } from '@kbn/es-query';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import { usePageUrlState, useUrlState } from '@kbn/ml-url-state';
-
 import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import type { SearchQueryLanguage } from '@kbn/ml-query-utils';
+
+import type { Category, SparkLinesPerCategory } from '../../../common/api/log_categorization/types';
+
 import { useDataSource } from '../../hooks/use_data_source';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
@@ -39,7 +41,7 @@ import {
 import { SearchPanel } from '../search_panel';
 import { PageHeader } from '../page_header';
 
-import type { EventRate, Category, SparkLinesPerCategory } from './use_categorize_request';
+import type { EventRate } from './use_categorize_request';
 import { useCategorizeRequest } from './use_categorize_request';
 import { CategoryTable } from './category_table';
 import { DocumentCountChart } from './document_count_chart';