[Search] Fixes EQL search strategy

x-pack/plugins/data_enhanced/common/search/es_search/es_search_rxjs_utils.ts

@@ -5,7 +5,8 @@
  */
 
 import { of, merge, timer, throwError } from 'rxjs';
-import { takeWhile, switchMap, expand, mergeMap, tap } from 'rxjs/operators';
+import { map, takeWhile, switchMap, expand, mergeMap, tap } from 'rxjs/operators';
+import { ApiResponse } from '@elastic/elasticsearch';
 
 import {
   doSearch,
@@ -35,6 +36,15 @@ export const doPartialSearch = <SearchResponse = any>(
     takeWhile((response) => !isCompleteResponse(response), true)
   );
 
+export const normalizeEqlResponse = <SearchResponse extends ApiResponse = ApiResponse>() =>
+  map<SearchResponse, SearchResponse>((eqlResponse) => ({
+    ...eqlResponse,
+    body: {
+      ...eqlResponse.body,
+      ...eqlResponse,
+    },
+  }));
+
 export const throwOnEsError = () =>
   mergeMap((r: IKibanaSearchResponse) =>
     isErrorResponse(r) ? merge(of(r), throwError(new AbortError())) : of(r)

x-pack/plugins/data_enhanced/server/search/eql_search_strategy.test.ts

@@ -22,6 +22,8 @@ const getMockEqlResponse = () => ({
       sequences: [],
     },
   },
+  meta: {},
+  statusCode: 200,
 });
 
 describe('EQL search strategy', () => {
@@ -193,5 +195,20 @@ describe('EQL search strategy', () => {
         expect(requestOptions).toEqual(expect.objectContaining({ ignore: [400] }));
       });
     });
+
+    describe('response', () => {
+      it('contains a rawResponse field containing the full search response', async () => {
+        const eqlSearch = await eqlSearchStrategyProvider(mockLogger);
+        const response = await eqlSearch
+          .search({ id: 'my-search-id', options: { ignore: [400] } }, {}, mockDeps)
+          .toPromise();
+
+        expect(response).toEqual(
+          expect.objectContaining({
+            rawResponse: expect.objectContaining(getMockEqlResponse()),
+          })
+        );
+      });
+    });
   });
 });

x-pack/plugins/data_enhanced/server/search/eql_search_strategy.ts

@@ -8,7 +8,10 @@ import type { Logger } from 'kibana/server';
 import type { ApiResponse } from '@elastic/elasticsearch';
 
 import { search } from '../../../../../src/plugins/data/server';
-import { doPartialSearch } from '../../common/search/es_search/es_search_rxjs_utils';
+import {
+  doPartialSearch,
+  normalizeEqlResponse,
+} from '../../common/search/es_search/es_search_rxjs_utils';
 import { getAsyncOptions, getDefaultSearchParams } from './get_default_search_params';
 
 import type { ISearchStrategy, IEsRawSearchResponse } from '../../../../../src/plugins/data/server';
@@ -64,7 +67,7 @@ export const eqlSearchStrategyProvider = (
         (response) => response.body.id,
         request.id,
         options
-      ).pipe(utils.toKibanaSearchResponse());
+      ).pipe(normalizeEqlResponse(), utils.toKibanaSearchResponse());
     },
   };
 };

x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts

@@ -50,6 +50,10 @@ export const EQL_TYPE = '[data-test-subj="eqlRuleType"]';
 
 export const EQL_QUERY_INPUT = '[data-test-subj="eqlQueryBarTextInput"]';
 
+export const EQL_QUERY_PREVIEW_HISTOGRAM = '[data-test-subj="queryPreviewEqlHistogram"]';
+
+export const EQL_QUERY_VALIDATION_SPINNER = '[data-test-subj="eql-validation-loading"]';
+
 export const IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK =
   '[data-test-subj="importQueryFromSavedTimeline"]';
 
@@ -80,6 +84,8 @@ export const MITRE_TACTIC_DROPDOWN = '[data-test-subj="mitreTactic"]';
 export const MITRE_TECHNIQUES_INPUT =
   '[data-test-subj="mitreTechniques"] [data-test-subj="comboBoxSearchInput"]';
 
+export const QUERY_PREVIEW_BUTTON = '[data-test-subj="queryPreviewButton"]';
+
 export const REFERENCE_URLS_INPUT =
   '[data-test-subj="detectionEngineStepAboutRuleReferenceUrls"] input';
 

x-pack/plugins/security_solution/cypress/screens/shared.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const NOTIFICATION_TOASTS = '[data-test-subj="globalToastList"]';
+
+export const TOAST_ERROR_CLASS = 'euiToast--danger';

x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts

@@ -61,7 +61,11 @@ import {
   THRESHOLD_TYPE,
   EQL_TYPE,
   EQL_QUERY_INPUT,
+  QUERY_PREVIEW_BUTTON,
+  EQL_QUERY_PREVIEW_HISTOGRAM,
+  EQL_QUERY_VALIDATION_SPINNER,
 } from '../screens/create_new_rule';
+import { NOTIFICATION_TOASTS, TOAST_ERROR_CLASS } from '../screens/shared';
 import { TIMELINE } from '../screens/timelines';
 import { refreshPage } from './security_header';
 
@@ -225,8 +229,12 @@ export const fillDefineThresholdRuleAndContinue = (rule: ThresholdRule) => {
 
 export const fillDefineEqlRuleAndContinue = (rule: CustomRule) => {
   cy.get(EQL_QUERY_INPUT).type(rule.customQuery);
-  cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true });
+  cy.get(EQL_QUERY_VALIDATION_SPINNER).should('not.exist');
+  cy.get(QUERY_PREVIEW_BUTTON).should('not.be.disabled').click({ force: true });
+  cy.get(EQL_QUERY_PREVIEW_HISTOGRAM).should('contain.text', 'Hits');
+  cy.get(NOTIFICATION_TOASTS).children().should('not.have.class', TOAST_ERROR_CLASS); // asserts no error toast on page
 
+  cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true });
   cy.get(EQL_QUERY_INPUT).should('not.exist');
 };
 

x-pack/plugins/security_solution/public/common/hooks/eql/api.ts

@@ -32,8 +32,7 @@ export const validateEql = async ({
   const { rawResponse: response } = await data.search
     .search<EqlSearchStrategyRequest, EqlSearchStrategyResponse>(
       {
-        // @ts-expect-error allow_no_indices is missing on EqlSearch
-        params: { allow_no_indices: true, index: index.join(), body: { query, size: 0 } },
+        params: { index: index.join(), body: { query, size: 0 } },
         options: { ignore: [400] },
       },
       {

x-pack/plugins/security_solution/public/common/hooks/eql/use_eql_preview.ts

@@ -74,8 +74,6 @@ export const useEqlPreview = (): [
           .search<EqlSearchStrategyRequest, EqlSearchStrategyResponse<EqlSearchResponse<Source>>>(
             {
               params: {
-                // @ts-expect-error allow_no_indices is missing on EqlSearch
-                allow_no_indices: true,
                 index: index.join(),
                 body: {
                   filter: {

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/custom_histogram.test.tsx

@@ -51,7 +51,7 @@ describe('PreviewCustomQueryHistogram', () => {
 
     expect(wrapper.find('[data-test-subj="queryPreviewLoading"]').exists()).toBeTruthy();
     expect(
-      wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').at(0).prop('subtitle')
+      wrapper.find('[dataTestSubj="queryPreviewCustomHistogram"]').at(0).prop('subtitle')
     ).toEqual(i18n.QUERY_PREVIEW_SUBTITLE_LOADING);
   });
 
@@ -78,32 +78,32 @@ describe('PreviewCustomQueryHistogram', () => {
 
     expect(wrapper.find('[data-test-subj="queryPreviewLoading"]').exists()).toBeFalsy();
     expect(
-      wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').at(0).prop('subtitle')
+      wrapper.find('[dataTestSubj="queryPreviewCustomHistogram"]').at(0).prop('subtitle')
     ).toEqual(i18n.QUERY_PREVIEW_TITLE(9154));
-    expect(
-      wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').at(0).props().data
-    ).toEqual([
-      {
-        key: 'hits',
-        value: [
-          {
-            g: 'All others',
-            x: 1602247050000,
-            y: 2314,
-          },
-          {
-            g: 'All others',
-            x: 1602247162500,
-            y: 3471,
-          },
-          {
-            g: 'All others',
-            x: 1602247275000,
-            y: 3369,
-          },
-        ],
-      },
-    ]);
+    expect(wrapper.find('[dataTestSubj="queryPreviewCustomHistogram"]').at(0).props().data).toEqual(
+      [
+        {
+          key: 'hits',
+          value: [
+            {
+              g: 'All others',
+              x: 1602247050000,
+              y: 2314,
+            },
+            {
+              g: 'All others',
+              x: 1602247162500,
+              y: 3471,
+            },
+            {
+              g: 'All others',
+              x: 1602247275000,
+              y: 3369,
+            },
+          ],
+        },
+      ]
+    );
   });
 
   test('it invokes setQuery with id, inspect, isLoading and refetch', async () => {

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/custom_histogram.tsx

@@ -69,7 +69,7 @@ export const PreviewCustomQueryHistogram = ({
       subtitle={subtitle}
       disclaimer={i18n.QUERY_PREVIEW_DISCLAIMER}
       isLoading={isLoading}
-      data-test-subj="queryPreviewCustomHistogram"
+      dataTestSubj="queryPreviewCustomHistogram"
     />
   );
 };

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.test.tsx

@@ -51,7 +51,7 @@ describe('PreviewEqlQueryHistogram', () => {
 
     expect(wrapper.find('[data-test-subj="queryPreviewLoading"]').exists()).toBeTruthy();
     expect(
-      wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').at(0).prop('subtitle')
+      wrapper.find('[dataTestSubj="queryPreviewEqlHistogram"]').at(0).prop('subtitle')
     ).toEqual(i18n.QUERY_PREVIEW_SUBTITLE_LOADING);
   });
 
@@ -78,9 +78,9 @@ describe('PreviewEqlQueryHistogram', () => {
 
     expect(wrapper.find('[data-test-subj="queryPreviewLoading"]').exists()).toBeFalsy();
     expect(
-      wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').at(0).prop('subtitle')
+      wrapper.find('[dataTestSubj="queryPreviewEqlHistogram"]').at(0).prop('subtitle')
     ).toEqual(i18n.QUERY_PREVIEW_TITLE(9154));
-    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').at(0).props().data).toEqual([
+    expect(wrapper.find('[dataTestSubj="queryPreviewEqlHistogram"]').at(0).props().data).toEqual([
       {
         key: 'hits',
         value: [

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.tsx

@@ -66,7 +66,7 @@ export const PreviewEqlQueryHistogram = ({
       subtitle={subtitle}
       disclaimer={i18n.QUERY_PREVIEW_DISCLAIMER_EQL}
       isLoading={isLoading}
-      data-test-subj="queryPreviewEqlHistogram"
+      dataTestSubj="queryPreviewEqlHistogram"
     />
   );
 };

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/histogram.tsx

@@ -21,6 +21,7 @@ const LoadingChart = styled(EuiLoadingChart)`
 interface PreviewHistogramProps {
   id: string;
   data: ChartSeriesData[];
+  dataTestSubj?: string;
   barConfig: ChartSeriesConfigs;
   title: string;
   subtitle: string;
@@ -31,6 +32,7 @@ interface PreviewHistogramProps {
 export const PreviewHistogram = ({
   id,
   data,
+  dataTestSubj,
   barConfig,
   title,
   subtitle,
@@ -39,7 +41,7 @@ export const PreviewHistogram = ({
 }: PreviewHistogramProps) => {
   return (
     <>
-      <Panel height={300}>
+      <Panel height={300} data-test-subj={dataTestSubj}>
         <EuiFlexGroup gutterSize="none" direction="column">
           <EuiFlexItem grow={1}>
             <HeaderSection id={id} title={title} titleSize="xs" subtitle={subtitle} />

x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx

@@ -79,9 +79,9 @@ describe('PreviewQuery', () => {
     expect(
       wrapper.find('[data-test-subj="queryPreviewButton"] button').props().disabled
     ).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it renders preview button disabled if "isDisabled" is true', () => {
@@ -146,9 +146,9 @@ describe('PreviewQuery', () => {
     const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
 
     expect(mockCalls.length).toEqual(1);
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeTruthy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it renders noise warning when rule type is query, timeframe is last hour and hit average is greater than 1/hour', async () => {
@@ -209,9 +209,9 @@ describe('PreviewQuery', () => {
     const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
 
     expect(mockCalls.length).toEqual(1);
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeTruthy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it renders eql histogram when preview button clicked and rule type is eql', () => {
@@ -236,9 +236,9 @@ describe('PreviewQuery', () => {
     const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
 
     expect(mockCalls.length).toEqual(1);
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeTruthy();
   });
 
   test('it renders noise warning when rule type is eql, timeframe is last hour and hit average is greater than 1/hour', async () => {
@@ -314,9 +314,9 @@ describe('PreviewQuery', () => {
 
     expect(mockCalls.length).toEqual(1);
     expect(wrapper.find('[data-test-subj="previewQueryWarning"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeTruthy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it renders noise warning when rule type is threshold, and threshold field is defined, timeframe is last hour and hit average is greater than 1/hour', async () => {
@@ -380,9 +380,9 @@ describe('PreviewQuery', () => {
     const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
 
     expect(mockCalls.length).toEqual(1);
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeTruthy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it renders query histogram when preview button clicked, rule type is threshold, and threshold field is empty string', () => {
@@ -407,9 +407,9 @@ describe('PreviewQuery', () => {
     const mockCalls = (useKibana().services.data.search.search as jest.Mock).mock.calls;
 
     expect(mockCalls.length).toEqual(1);
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeTruthy();
-    expect(wrapper.find('[data-test-subj="previewThresholdQueryHistogram"]').exists()).toBeFalsy();
-    expect(wrapper.find('[data-test-subj="previewEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="thresholdQueryPreviewHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewEqlHistogram"]').exists()).toBeFalsy();
   });
 
   test('it hides histogram when timeframe changes', () => {
@@ -431,13 +431,13 @@ describe('PreviewQuery', () => {
 
     wrapper.find('[data-test-subj="queryPreviewButton"] button').at(0).simulate('click');
 
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeTruthy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeTruthy();
 
     wrapper
       .find('[data-test-subj="queryPreviewTimeframeSelect"] select')
       .at(0)
       .simulate('change', { target: { value: 'd' } });
 
-    expect(wrapper.find('[data-test-subj="previewNonEqlQueryHistogram"]').exists()).toBeFalsy();
+    expect(wrapper.find('[data-test-subj="queryPreviewCustomHistogram"]').exists()).toBeFalsy();
   });
 });