[data.search] Allow search response to follow new hits format (#88115) (

#89951)

* [data.search] Allow search response to follow new hits format

* Update docs

* Fix types

* Fix types

* Doc updates

* Remove declare module

* Remove declare module

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md)
+
+## ISearchOptions.legacyHitsTotal property
+
+Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to something other than `true`<!-- -->, this should be set to `false`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+legacyHitsTotal?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -17,6 +17,7 @@ export interface ISearchOptions
 |  [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
+|  [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iscopedsessionservice.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IScopedSessionService](./kibana-plugin-plugins-data-server.iscopedsessionservice.md)
+
+## IScopedSessionService interface
+
+<b>Signature:</b>
+
+```typescript
+export interface IScopedSessionService 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [search](./kibana-plugin-plugins-data-server.iscopedsessionservice.search.md) | <code>&lt;Request extends IKibanaSearchRequest, Response extends IKibanaSearchResponse&gt;(strategy: ISearchStrategy&lt;Request, Response&gt;, ...args: Parameters&lt;ISearchStrategy&lt;Request, Response&gt;['search']&gt;) =&gt; Observable&lt;Response&gt;</code> |  |
+

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.iscopedsessionservice.search.md

@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [IScopedSessionService](./kibana-plugin-plugins-data-server.iscopedsessionservice.md) &gt; [search](./kibana-plugin-plugins-data-server.iscopedsessionservice.search.md)
+
+## IScopedSessionService.search property
+
+<b>Signature:</b>
+
+```typescript
+search: <Request extends IKibanaSearchRequest, Response extends IKibanaSearchResponse>(strategy: ISearchStrategy<Request, Response>, ...args: Parameters<ISearchStrategy<Request, Response>['search']>) => Observable<Response>;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md)
+
+## ISearchOptions.legacyHitsTotal property
+
+Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to something other than `true`<!-- -->, this should be set to `false`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+legacyHitsTotal?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -17,6 +17,7 @@ export interface ISearchOptions
 |  [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
+|  [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md

@@ -53,6 +53,7 @@
 |  [IFieldSubType](./kibana-plugin-plugins-data-server.ifieldsubtype.md) |  |
 |  [IFieldType](./kibana-plugin-plugins-data-server.ifieldtype.md) |  |
 |  [IndexPatternAttributes](./kibana-plugin-plugins-data-server.indexpatternattributes.md) | Interface for an index pattern saved object |
+|  [IScopedSessionService](./kibana-plugin-plugins-data-server.iscopedsessionservice.md) |  |
 |  [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) |  |
 |  [ISearchSetup](./kibana-plugin-plugins-data-server.isearchsetup.md) |  |
 |  [ISearchStart](./kibana-plugin-plugins-data-server.isearchstart.md) |  |

src/plugins/data/common/search/types.ts

@@ -84,11 +84,18 @@ export interface ISearchOptions {
    * An `AbortSignal` that allows the caller of `search` to abort a search request.
    */
   abortSignal?: AbortSignal;
+
   /**
    * Use this option to force using a specific server side search strategy. Leave empty to use the default strategy.
    */
   strategy?: string;
 
+  /**
+   * Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to
+   * something other than `true`, this should be set to `false`.
+   */
+  legacyHitsTotal?: boolean;
+
   /**
    * A session ID, grouping multiple search requests into a single session.
    */

src/plugins/data/public/public.api.md

@@ -1649,6 +1649,7 @@ export interface ISearchOptions {
     abortSignal?: AbortSignal;
     isRestore?: boolean;
     isStored?: boolean;
+    legacyHitsTotal?: boolean;
     sessionId?: string;
     strategy?: string;
 }

src/plugins/data/server/index.ts

@@ -235,6 +235,7 @@ export {
   SearchUsage,
   SessionService,
   ISessionService,
+  IScopedSessionService,
   DataApiRequestHandlerContext,
   DataRequestHandlerContext,
 } from './search';

src/plugins/data/server/search/es_search/es_search_strategy.ts

@@ -13,7 +13,7 @@ import type { Logger, SharedGlobalConfig } from 'kibana/server';
 import type { ISearchStrategy } from '../types';
 import type { SearchUsage } from '../collectors';
 import { getDefaultSearchParams, getShardTimeout, shimAbortSignal } from './request_utils';
-import { toKibanaSearchResponse } from './response_utils';
+import { shimHitsTotal, toKibanaSearchResponse } from './response_utils';
 import { searchUsageObserver } from '../collectors/usage';
 import { getKbnServerError, KbnServerError } from '../../../../kibana_utils/server';
 
@@ -29,7 +29,7 @@ export const esSearchStrategyProvider = (
    * @throws `KbnServerError`
    * @returns `Observable<IEsSearchResponse<any>>`
    */
-  search: (request, { abortSignal }, { esClient, uiSettingsClient }) => {
+  search: (request, { abortSignal, ...options }, { esClient, uiSettingsClient }) => {
     // Only default index pattern type is supported here.
     // See data_enhanced for other type support.
     if (request.indexType) {
@@ -46,7 +46,8 @@ export const esSearchStrategyProvider = (
         };
         const promise = esClient.asCurrentUser.search<SearchResponse<unknown>>(params);
         const { body } = await shimAbortSignal(promise, abortSignal);
-        return toKibanaSearchResponse(body);
+        const response = shimHitsTotal(body, options);
+        return toKibanaSearchResponse(response);
       } catch (e) {
         throw getKbnServerError(e);
       }

src/plugins/data/server/search/es_search/response_utils.test.ts

@@ -6,7 +6,7 @@
  * Public License, v 1.
  */
 
-import { getTotalLoaded, toKibanaSearchResponse } from './response_utils';
+import { getTotalLoaded, toKibanaSearchResponse, shimHitsTotal } from './response_utils';
 import { SearchResponse } from 'elasticsearch';
 
 describe('response utils', () => {
@@ -55,4 +55,79 @@ describe('response utils', () => {
       });
     });
   });
+
+  describe('shimHitsTotal', () => {
+    test('returns the total if it is already numeric', () => {
+      const result = shimHitsTotal({
+        hits: {
+          total: 5,
+        },
+      } as any);
+      expect(result).toEqual({
+        hits: {
+          total: 5,
+        },
+      });
+    });
+
+    test('returns the total if it is inside `value`', () => {
+      const result = shimHitsTotal({
+        hits: {
+          total: {
+            value: 5,
+          },
+        },
+      } as any);
+      expect(result).toEqual({
+        hits: {
+          total: 5,
+        },
+      });
+    });
+
+    test('returns other properties from the response', () => {
+      const result = shimHitsTotal({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: {
+            value: 5,
+          },
+        },
+      } as any);
+      expect(result).toEqual({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: 5,
+        },
+      });
+    });
+
+    test('returns the response as-is if `legacyHitsTotal` is `false`', () => {
+      const result = shimHitsTotal(
+        {
+          _shards: {},
+          hits: {
+            hits: [],
+            total: {
+              value: 5,
+              relation: 'eq',
+            },
+          },
+        } as any,
+        { legacyHitsTotal: false }
+      );
+      expect(result).toEqual({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: {
+            value: 5,
+            relation: 'eq',
+          },
+        },
+      });
+    });
+  });
 });

src/plugins/data/server/search/es_search/response_utils.ts

@@ -7,6 +7,7 @@
  */
 
 import { SearchResponse } from 'elasticsearch';
+import { ISearchOptions } from '../../../common';
 
 /**
  * Get the `total`/`loaded` for this response (see `IKibanaSearchResponse`). Note that `skipped` is
@@ -31,3 +32,20 @@ export function toKibanaSearchResponse(rawResponse: SearchResponse<unknown>) {
     ...getTotalLoaded(rawResponse),
   };
 }
+
+/**
+ * Temporary workaround until https://github.com/elastic/kibana/issues/26356 is addressed.
+ * Since we are setting `track_total_hits` in the request, `hits.total` will be an object
+ * containing the `value`.
+ *
+ * @internal
+ */
+export function shimHitsTotal(
+  response: SearchResponse<unknown>,
+  { legacyHitsTotal = true }: ISearchOptions = {}
+) {
+  if (!legacyHitsTotal) return response;
+  const total = (response.hits?.total as any)?.value ?? response.hits?.total;
+  const hits = { ...response.hits, total };
+  return { ...response, hits };
+}

src/plugins/data/server/search/index.ts

@@ -10,5 +10,4 @@ export * from './types';
 export * from './es_search';
 export { usageProvider, SearchUsage, searchUsageObserver } from './collectors';
 export * from './aggs';
-export { shimHitsTotal } from './routes';
 export * from './session';

src/plugins/data/server/search/routes/bsearch.ts

@@ -6,7 +6,7 @@
  * Public License, v 1.
  */
 
-import { catchError, first, map } from 'rxjs/operators';
+import { catchError, first } from 'rxjs/operators';
 import { CoreStart, KibanaRequest } from 'src/core/server';
 import { BfetchServerSetup } from 'src/plugins/bfetch/server';
 import {
@@ -15,7 +15,6 @@ import {
   ISearchClient,
   ISearchOptions,
 } from '../../../common/search';
-import { shimHitsTotal } from './shim_hits_total';
 
 type GetScopedProider = (coreStart: CoreStart) => (request: KibanaRequest) => ISearchClient;
 
@@ -40,14 +39,6 @@ export function registerBsearchRoute(
           .search(requestData, options)
           .pipe(
             first(),
-            map((response) => {
-              return {
-                ...response,
-                ...{
-                  rawResponse: shimHitsTotal(response.rawResponse),
-                },
-              };
-            }),
             catchError((err) => {
               // Re-throw as object, to get attributes passed to the client
               // eslint-disable-next-line no-throw-literal

src/plugins/data/server/search/routes/call_msearch.ts

@@ -12,9 +12,8 @@ import { SearchResponse } from 'elasticsearch';
 import { IUiSettingsClient, IScopedClusterClient, SharedGlobalConfig } from 'src/core/server';
 
 import type { MsearchRequestBody, MsearchResponse } from '../../../common/search/search_source';
-import { shimHitsTotal } from './shim_hits_total';
 import { getKbnServerError } from '../../../../kibana_utils/server';
-import { getShardTimeout, getDefaultSearchParams, shimAbortSignal } from '..';
+import { getShardTimeout, getDefaultSearchParams, shimAbortSignal, shimHitsTotal } from '..';
 
 /** @internal */
 export function convertRequestBody(

src/plugins/data/server/search/routes/index.ts

@@ -9,4 +9,3 @@
 export * from './call_msearch';
 export * from './msearch';
 export * from './search';
-export * from './shim_hits_total';

src/plugins/data/server/search/routes/search.ts

@@ -9,7 +9,6 @@
 import { first } from 'rxjs/operators';
 import { schema } from '@kbn/config-schema';
 import { getRequestAbortedSignal } from '../../lib';
-import { shimHitsTotal } from './shim_hits_total';
 import { reportServerError } from '../../../../kibana_utils/server';
 import type { DataPluginRouter } from '../types';
 
@@ -27,6 +26,7 @@ export function registerSearchRoute(router: DataPluginRouter): void {
 
         body: schema.object(
           {
+            legacyHitsTotal: schema.maybe(schema.boolean()),
             sessionId: schema.maybe(schema.string()),
             isStored: schema.maybe(schema.boolean()),
             isRestore: schema.maybe(schema.boolean()),
@@ -36,7 +36,13 @@ export function registerSearchRoute(router: DataPluginRouter): void {
       },
     },
     async (context, request, res) => {
-      const { sessionId, isStored, isRestore, ...searchRequest } = request.body;
+      const {
+        legacyHitsTotal = true,
+        sessionId,
+        isStored,
+        isRestore,
+        ...searchRequest
+      } = request.body;
       const { strategy, id } = request.params;
       const abortSignal = getRequestAbortedSignal(request.events.aborted$);
 
@@ -47,6 +53,7 @@ export function registerSearchRoute(router: DataPluginRouter): void {
             {
               abortSignal,
               strategy,
+              legacyHitsTotal,
               sessionId,
               isStored,
               isRestore,
@@ -55,14 +62,7 @@ export function registerSearchRoute(router: DataPluginRouter): void {
           .pipe(first())
           .toPromise();
 
-        return res.ok({
-          body: {
-            ...response,
-            ...{
-              rawResponse: shimHitsTotal(response.rawResponse),
-            },
-          },
-        });
+        return res.ok({ body: response });
       } catch (err) {
         return reportServerError(res, err);
       }