[data.search] Allow search response to follow new hits format

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-public](./kibana-plugin-plugins-data-public.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-public.isearchoptions.md) &gt; [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md)
+
+## ISearchOptions.legacyHitsTotal property
+
+Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to something other than `true`<!-- -->, this should be set to `false`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+legacyHitsTotal?: boolean;
+```

docs/development/plugins/data/public/kibana-plugin-plugins-data-public.isearchoptions.md

@@ -17,6 +17,7 @@ export interface ISearchOptions
 |  [abortSignal](./kibana-plugin-plugins-data-public.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [isRestore](./kibana-plugin-plugins-data-public.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-public.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
+|  [legacyHitsTotal](./kibana-plugin-plugins-data-public.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
 |  [sessionId](./kibana-plugin-plugins-data-public.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-public.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md

@@ -0,0 +1,13 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-plugins-data-server](./kibana-plugin-plugins-data-server.md) &gt; [ISearchOptions](./kibana-plugin-plugins-data-server.isearchoptions.md) &gt; [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md)
+
+## ISearchOptions.legacyHitsTotal property
+
+Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to something other than `true`<!-- -->, this should be set to `false`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+legacyHitsTotal?: boolean;
+```

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.isearchoptions.md

@@ -17,6 +17,7 @@ export interface ISearchOptions
 |  [abortSignal](./kibana-plugin-plugins-data-server.isearchoptions.abortsignal.md) | <code>AbortSignal</code> | An <code>AbortSignal</code> that allows the caller of <code>search</code> to abort a search request. |
 |  [isRestore](./kibana-plugin-plugins-data-server.isearchoptions.isrestore.md) | <code>boolean</code> | Whether the session is restored (i.e. search requests should re-use the stored search IDs, rather than starting from scratch) |
 |  [isStored](./kibana-plugin-plugins-data-server.isearchoptions.isstored.md) | <code>boolean</code> | Whether the session is already saved (i.e. sent to background) |
+|  [legacyHitsTotal](./kibana-plugin-plugins-data-server.isearchoptions.legacyhitstotal.md) | <code>boolean</code> | Request the legacy format for the total number of hits. If sending <code>rest_total_hits_as_int</code> to something other than <code>true</code>, this should be set to <code>false</code>. |
 |  [sessionId](./kibana-plugin-plugins-data-server.isearchoptions.sessionid.md) | <code>string</code> | A session ID, grouping multiple search requests into a single session. |
 |  [strategy](./kibana-plugin-plugins-data-server.isearchoptions.strategy.md) | <code>string</code> | Use this option to force using a specific server side search strategy. Leave empty to use the default strategy. |
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.md

@@ -14,7 +14,7 @@
 |  [IndexPatternsService](./kibana-plugin-plugins-data-server.indexpatternsservice.md) |  |
 |  [OptionedParamType](./kibana-plugin-plugins-data-server.optionedparamtype.md) |  |
 |  [Plugin](./kibana-plugin-plugins-data-server.plugin.md) |  |
-|  [SessionService](./kibana-plugin-plugins-data-server.sessionservice.md) | The OSS session service. See data\_enhanced in X-Pack for the background session service. |
+|  [SessionService](./kibana-plugin-plugins-data-server.sessionservice.md) | The OSS session service. See data\_enhanced in X-Pack for the search session service. |
 
 ## Enumerations
 

docs/development/plugins/data/server/kibana-plugin-plugins-data-server.sessionservice.md

@@ -4,7 +4,7 @@
 
 ## SessionService class
 
-The OSS session service. See data\_enhanced in X-Pack for the background session service.
+The OSS session service. See data\_enhanced in X-Pack for the search session service.
 
 <b>Signature:</b>
 

src/plugins/data/common/search/types.ts

@@ -95,11 +95,18 @@ export interface ISearchOptions {
    * An `AbortSignal` that allows the caller of `search` to abort a search request.
    */
   abortSignal?: AbortSignal;
+
   /**
    * Use this option to force using a specific server side search strategy. Leave empty to use the default strategy.
    */
   strategy?: string;
 
+  /**
+   * Request the legacy format for the total number of hits. If sending `rest_total_hits_as_int` to
+   * something other than `true`, this should be set to `false`.
+   */
+  legacyHitsTotal?: boolean;
+
   /**
    * A session ID, grouping multiple search requests into a single session.
    */

src/plugins/data/public/public.api.md

@@ -1632,6 +1632,7 @@ export interface ISearchOptions {
     abortSignal?: AbortSignal;
     isRestore?: boolean;
     isStored?: boolean;
+    legacyHitsTotal?: boolean;
     sessionId?: string;
     strategy?: string;
 }

src/plugins/data/server/search/es_search/es_search_strategy.ts

@@ -23,7 +23,7 @@ import type { Logger, SharedGlobalConfig } from 'kibana/server';
 import type { ISearchStrategy } from '../types';
 import type { SearchUsage } from '../collectors';
 import { getDefaultSearchParams, getShardTimeout, shimAbortSignal } from './request_utils';
-import { toKibanaSearchResponse } from './response_utils';
+import { shimHitsTotal, toKibanaSearchResponse } from './response_utils';
 import { searchUsageObserver } from '../collectors/usage';
 import { KbnServerError } from '../../../../kibana_utils/server';
 
@@ -32,7 +32,7 @@ export const esSearchStrategyProvider = (
   logger: Logger,
   usage?: SearchUsage
 ): ISearchStrategy => ({
-  search: (request, { abortSignal }, { esClient, uiSettingsClient }) => {
+  search: (request, { abortSignal, ...options }, { esClient, uiSettingsClient }) => {
     // Only default index pattern type is supported here.
     // See data_enhanced for other type support.
     if (request.indexType) {
@@ -48,7 +48,8 @@ export const esSearchStrategyProvider = (
       };
       const promise = esClient.asCurrentUser.search<SearchResponse<unknown>>(params);
       const { body } = await shimAbortSignal(promise, abortSignal);
-      return toKibanaSearchResponse(body);
+      const response = shimHitsTotal(body, options);
+      return toKibanaSearchResponse(response);
     };
 
     return from(search()).pipe(tap(searchUsageObserver(logger, usage)));

src/plugins/data/server/search/es_search/response_utils.test.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import { getTotalLoaded, toKibanaSearchResponse } from './response_utils';
+import { getTotalLoaded, toKibanaSearchResponse, shimHitsTotal } from './response_utils';
 import { SearchResponse } from 'elasticsearch';
 
 describe('response utils', () => {
@@ -66,4 +66,79 @@ describe('response utils', () => {
       });
     });
   });
+
+  describe('shimHitsTotal', () => {
+    test('returns the total if it is already numeric', () => {
+      const result = shimHitsTotal({
+        hits: {
+          total: 5,
+        },
+      } as any);
+      expect(result).toEqual({
+        hits: {
+          total: 5,
+        },
+      });
+    });
+
+    test('returns the total if it is inside `value`', () => {
+      const result = shimHitsTotal({
+        hits: {
+          total: {
+            value: 5,
+          },
+        },
+      } as any);
+      expect(result).toEqual({
+        hits: {
+          total: 5,
+        },
+      });
+    });
+
+    test('returns other properties from the response', () => {
+      const result = shimHitsTotal({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: {
+            value: 5,
+          },
+        },
+      } as any);
+      expect(result).toEqual({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: 5,
+        },
+      });
+    });
+
+    test('returns the response as-is if `legacyHitsTotal` is `false`', () => {
+      const result = shimHitsTotal(
+        {
+          _shards: {},
+          hits: {
+            hits: [],
+            total: {
+              value: 5,
+              relation: 'eq',
+            },
+          },
+        } as any,
+        { legacyHitsTotal: false }
+      );
+      expect(result).toEqual({
+        _shards: {},
+        hits: {
+          hits: [],
+          total: {
+            value: 5,
+            relation: 'eq',
+          },
+        },
+      });
+    });
+  });
 });

src/plugins/data/server/search/es_search/response_utils.ts

@@ -18,6 +18,7 @@
  */
 
 import { SearchResponse } from 'elasticsearch';
+import { ISearchOptions } from '../../../common';
 
 /**
  * Get the `total`/`loaded` for this response (see `IKibanaSearchResponse`). Note that `skipped` is
@@ -42,3 +43,20 @@ export function toKibanaSearchResponse(rawResponse: SearchResponse<unknown>) {
     ...getTotalLoaded(rawResponse),
   };
 }
+
+/**
+ * Temporary workaround until https://github.com/elastic/kibana/issues/26356 is addressed.
+ * Since we are setting `track_total_hits` in the request, `hits.total` will be an object
+ * containing the `value`.
+ *
+ * @internal
+ */
+export function shimHitsTotal(
+  response: SearchResponse<unknown>,
+  { legacyHitsTotal = true }: ISearchOptions = {}
+) {
+  if (!legacyHitsTotal) return response;
+  const total = (response.hits?.total as any)?.value ?? response.hits?.total;
+  const hits = { ...response.hits, total };
+  return { ...response, hits };
+}

src/plugins/data/server/search/index.ts

@@ -21,5 +21,4 @@ export * from './types';
 export * from './es_search';
 export { usageProvider, SearchUsage, searchUsageObserver } from './collectors';
 export * from './aggs';
-export { shimHitsTotal } from './routes';
 export * from './session';

src/plugins/data/server/search/routes/call_msearch.ts

@@ -24,8 +24,7 @@ import { SearchResponse } from 'elasticsearch';
 import { IUiSettingsClient, IScopedClusterClient, SharedGlobalConfig } from 'src/core/server';
 
 import type { MsearchRequestBody, MsearchResponse } from '../../../common/search/search_source';
-import { shimHitsTotal } from './shim_hits_total';
-import { getShardTimeout, getDefaultSearchParams, shimAbortSignal } from '..';
+import { getShardTimeout, getDefaultSearchParams, shimAbortSignal, shimHitsTotal } from '..';
 
 /** @internal */
 export function convertRequestBody(

src/plugins/data/server/search/routes/index.ts

@@ -20,4 +20,3 @@
 export * from './call_msearch';
 export * from './msearch';
 export * from './search';
-export * from './shim_hits_total';

src/plugins/data/server/search/routes/search.ts

@@ -21,7 +21,6 @@ import { first } from 'rxjs/operators';
 import { schema } from '@kbn/config-schema';
 import type { IRouter } from 'src/core/server';
 import { getRequestAbortedSignal } from '../../lib';
-import { shimHitsTotal } from './shim_hits_total';
 import { reportServerError } from '../../../../kibana_utils/server';
 
 export function registerSearchRoute(router: IRouter): void {
@@ -38,6 +37,7 @@ export function registerSearchRoute(router: IRouter): void {
 
         body: schema.object(
           {
+            legacyHitsTotal: schema.maybe(schema.boolean()),
             sessionId: schema.maybe(schema.string()),
             isStored: schema.maybe(schema.boolean()),
             isRestore: schema.maybe(schema.boolean()),
@@ -47,7 +47,13 @@ export function registerSearchRoute(router: IRouter): void {
       },
     },
     async (context, request, res) => {
-      const { sessionId, isStored, isRestore, ...searchRequest } = request.body;
+      const {
+        legacyHitsTotal = true,
+        sessionId,
+        isStored,
+        isRestore,
+        ...searchRequest
+      } = request.body;
       const { strategy, id } = request.params;
       const abortSignal = getRequestAbortedSignal(request.events.aborted$);
 
@@ -58,6 +64,7 @@ export function registerSearchRoute(router: IRouter): void {
             {
               abortSignal,
               strategy,
+              legacyHitsTotal,
               sessionId,
               isStored,
               isRestore,
@@ -66,14 +73,7 @@ export function registerSearchRoute(router: IRouter): void {
           .pipe(first())
           .toPromise();
 
-        return res.ok({
-          body: {
-            ...response,
-            ...{
-              rawResponse: shimHitsTotal(response.rawResponse),
-            },
-          },
-        });
+        return res.ok({ body: response });
       } catch (err) {
         return reportServerError(res, err);
       }