Manually building KueryNode for Fleet's routes (#75693)

* Simple benchmark tests for kuery

* Building manually is "better" still not free

* Building the KueryNode manually

* Removing benchmark tests

* Another query is building the KueryNode manually

* Empty strings are inherently falsy

* No longer reaching into the data plugin, import from the "root" indexes

* Using AGENT_ACTION_SAVED_OBJECT_TYPE everywhere

* Adding SavedObjectsRepository#find unit test for KueryNode

* Adding KQL KueryNode test for validateConvertFilterToKueryNode

* /s/KQL string/KQL expression

* Updating API docs

* Adding micro benchmark

* Revert "Adding micro benchmark"

This reverts commit 97e19c0.

* Adding an empty string filters test

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.filter.md

@@ -7,5 +7,5 @@
 <b>Signature:</b>
 
 ```typescript
-filter?: string;
+filter?: string | KueryNode;
 ```

docs/development/core/public/kibana-plugin-core-public.savedobjectsfindoptions.md

@@ -17,7 +17,7 @@ export interface SavedObjectsFindOptions
 |  --- | --- | --- |
 |  [defaultSearchOperator](./kibana-plugin-core-public.savedobjectsfindoptions.defaultsearchoperator.md) | <code>'AND' &#124; 'OR'</code> |  |
 |  [fields](./kibana-plugin-core-public.savedobjectsfindoptions.fields.md) | <code>string[]</code> | An array of fields to include in the results |
-|  [filter](./kibana-plugin-core-public.savedobjectsfindoptions.filter.md) | <code>string</code> |  |
+|  [filter](./kibana-plugin-core-public.savedobjectsfindoptions.filter.md) | <code>string &#124; KueryNode</code> |  |
 |  [hasReference](./kibana-plugin-core-public.savedobjectsfindoptions.hasreference.md) | <code>{</code><br/><code>        type: string;</code><br/><code>        id: string;</code><br/><code>    }</code> |  |
 |  [namespaces](./kibana-plugin-core-public.savedobjectsfindoptions.namespaces.md) | <code>string[]</code> |  |
 |  [page](./kibana-plugin-core-public.savedobjectsfindoptions.page.md) | <code>number</code> |  |

docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.filter.md

@@ -7,5 +7,5 @@
 <b>Signature:</b>
 
 ```typescript
-filter?: string;
+filter?: string | KueryNode;
 ```

docs/development/core/server/kibana-plugin-core-server.savedobjectsfindoptions.md

@@ -17,7 +17,7 @@ export interface SavedObjectsFindOptions
 |  --- | --- | --- |
 |  [defaultSearchOperator](./kibana-plugin-core-server.savedobjectsfindoptions.defaultsearchoperator.md) | <code>'AND' &#124; 'OR'</code> |  |
 |  [fields](./kibana-plugin-core-server.savedobjectsfindoptions.fields.md) | <code>string[]</code> | An array of fields to include in the results |
-|  [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | <code>string</code> |  |
+|  [filter](./kibana-plugin-core-server.savedobjectsfindoptions.filter.md) | <code>string &#124; KueryNode</code> |  |
 |  [hasReference](./kibana-plugin-core-server.savedobjectsfindoptions.hasreference.md) | <code>{</code><br/><code>        type: string;</code><br/><code>        id: string;</code><br/><code>    }</code> |  |
 |  [namespaces](./kibana-plugin-core-server.savedobjectsfindoptions.namespaces.md) | <code>string[]</code> |  |
 |  [page](./kibana-plugin-core-server.savedobjectsfindoptions.page.md) | <code>number</code> |  |

src/core/public/public.api.md

@@ -1189,8 +1189,10 @@ export interface SavedObjectsFindOptions {
     // (undocumented)
     defaultSearchOperator?: 'AND' | 'OR';
     fields?: string[];
+    // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts
+    //
     // (undocumented)
-    filter?: string;
+    filter?: string | KueryNode;
     // (undocumented)
     hasReference?: {
         type: string;

src/core/server/saved_objects/service/lib/filter_utils.test.ts

@@ -83,7 +83,19 @@ const mockMappings = {
 
 describe('Filter Utils', () => {
   describe('#validateConvertFilterToKueryNode', () => {
-    test('Validate a simple filter', () => {
+    test('Empty string filters are ignored', () => {
+      expect(validateConvertFilterToKueryNode(['foo'], '', mockMappings)).toBeUndefined();
+    });
+    test('Validate a simple KQL KueryNode filter', () => {
+      expect(
+        validateConvertFilterToKueryNode(
+          ['foo'],
+          esKuery.nodeTypes.function.buildNode('is', `foo.attributes.title`, 'best', true),
+          mockMappings
+        )
+      ).toEqual(esKuery.fromKueryExpression('foo.title: "best"'));
+    });
+    test('Validate a simple KQL expression filter', () => {
       expect(
         validateConvertFilterToKueryNode(['foo'], 'foo.attributes.title: "best"', mockMappings)
       ).toEqual(esKuery.fromKueryExpression('foo.title: "best"'));

src/core/server/saved_objects/service/lib/filter_utils.ts

@@ -28,11 +28,12 @@ const astFunctionType = ['is', 'range', 'nested'];
 
 export const validateConvertFilterToKueryNode = (
   allowedTypes: string[],
-  filter: string,
+  filter: string | KueryNode,
   indexMapping: IndexMapping
 ): KueryNode | undefined => {
-  if (filter && filter.length > 0 && indexMapping) {
-    const filterKueryNode = esKuery.fromKueryExpression(filter);
+  if (filter && indexMapping) {
+    const filterKueryNode =
+      typeof filter === 'string' ? esKuery.fromKueryExpression(filter) : filter;
 
     const validationFilterKuery = validateFilterKueryNode({
       astFilter: filterKueryNode,

src/core/server/saved_objects/service/lib/repository.test.js

@@ -25,6 +25,8 @@ import { encodeHitVersion } from '../../version';
 import { SavedObjectTypeRegistry } from '../../saved_objects_type_registry';
 import { DocumentMigrator } from '../../migrations/core/document_migrator';
 import { elasticsearchClientMock } from '../../../elasticsearch/client/mocks';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { nodeTypes } from '../../../../../plugins/data/common/es_query';
 
 jest.mock('./search_dsl/search_dsl', () => ({ getSearchDsl: jest.fn() }));
 
@@ -2529,7 +2531,7 @@ describe('SavedObjectsRepository', () => {
         expect(getSearchDslNS.getSearchDsl).toHaveBeenCalledWith(mappings, registry, relevantOpts);
       });
 
-      it(`accepts KQL filter and passes kueryNode to getSearchDsl`, async () => {
+      it(`accepts KQL expression filter and passes KueryNode to getSearchDsl`, async () => {
         const findOpts = {
           namespace,
           search: 'foo*',
@@ -2570,6 +2572,47 @@ describe('SavedObjectsRepository', () => {
         `);
       });
 
+      it(`accepts KQL KueryNode filter and passes KueryNode to getSearchDsl`, async () => {
+        const findOpts = {
+          namespace,
+          search: 'foo*',
+          searchFields: ['foo'],
+          type: ['dashboard'],
+          sortField: 'name',
+          sortOrder: 'desc',
+          defaultSearchOperator: 'AND',
+          hasReference: {
+            type: 'foo',
+            id: '1',
+          },
+          indexPattern: undefined,
+          filter: nodeTypes.function.buildNode('is', `dashboard.attributes.otherField`, '*'),
+        };
+
+        await findSuccess(findOpts, namespace);
+        const { kueryNode } = getSearchDslNS.getSearchDsl.mock.calls[0][2];
+        expect(kueryNode).toMatchInlineSnapshot(`
+          Object {
+            "arguments": Array [
+              Object {
+                "type": "literal",
+                "value": "dashboard.otherField",
+              },
+              Object {
+                "type": "wildcard",
+                "value": "@kuery-wildcard@",
+              },
+              Object {
+                "type": "literal",
+                "value": false,
+              },
+            ],
+            "function": "is",
+            "type": "function",
+          }
+        `);
+      });
+
       it(`supports multiple types`, async () => {
         const types = ['config', 'index-pattern'];
         await findSuccess({ type: types });

src/core/server/saved_objects/types.ts

@@ -39,6 +39,9 @@ import { SavedObjectUnsanitizedDoc } from './serialization';
 import { SavedObjectsMigrationLogger } from './migrations/core/migration_logger';
 import { SavedObject } from '../../types';
 
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { KueryNode } from '../../../plugins/data/common';
+
 export {
   SavedObjectAttributes,
   SavedObjectAttribute,
@@ -89,7 +92,7 @@ export interface SavedObjectsFindOptions {
   rootSearchFields?: string[];
   hasReference?: { type: string; id: string };
   defaultSearchOperator?: 'AND' | 'OR';
-  filter?: string;
+  filter?: string | KueryNode;
   namespaces?: string[];
   /** An optional ES preference value to be used for the query **/
   preference?: string;

src/core/server/server.api.md

@@ -2320,8 +2320,10 @@ export interface SavedObjectsFindOptions {
     // (undocumented)
     defaultSearchOperator?: 'AND' | 'OR';
     fields?: string[];
+    // Warning: (ae-forgotten-export) The symbol "KueryNode" needs to be exported by the entry point index.d.ts
+    //
     // (undocumented)
-    filter?: string;
+    filter?: string | KueryNode;
     // (undocumented)
     hasReference?: {
         type: string;

x-pack/plugins/ingest_manager/server/services/agents/actions.ts

@@ -9,6 +9,7 @@ import { Agent, AgentAction, AgentActionSOAttributes } from '../../../common/typ
 import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../../common/constants';
 import { savedObjectToAgentAction } from './saved_objects';
 import { appContextService } from '../app_context';
+import { nodeTypes } from '../../../../../../src/plugins/data/common';
 
 export async function createAgentAction(
   soClient: SavedObjectsClientContract,
@@ -29,9 +30,24 @@ export async function getAgentActionsForCheckin(
   soClient: SavedObjectsClientContract,
   agentId: string
 ): Promise<AgentAction[]> {
+  const filter = nodeTypes.function.buildNode('and', [
+    nodeTypes.function.buildNode(
+      'not',
+      nodeTypes.function.buildNode(
+        'is',
+        `${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at`,
+        '*'
+      )
+    ),
+    nodeTypes.function.buildNode(
+      'is',
+      `${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.agent_id`,
+      agentId
+    ),
+  ]);
   const res = await soClient.find<AgentActionSOAttributes>({
     type: AGENT_ACTION_SAVED_OBJECT_TYPE,
-    filter: `not ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at: * and ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.agent_id:${agentId}`,
+    filter,
   });
 
   return Promise.all(
@@ -78,9 +94,26 @@ export async function getAgentActionByIds(
 }
 
 export async function getNewActionsSince(soClient: SavedObjectsClientContract, timestamp: string) {
+  const filter = nodeTypes.function.buildNode('and', [
+    nodeTypes.function.buildNode(
+      'not',
+      nodeTypes.function.buildNode(
+        'is',
+        `${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at`,
+        '*'
+      )
+    ),
+    nodeTypes.function.buildNode(
+      'range',
+      `${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.created_at`,
+      {
+        gte: timestamp,
+      }
+    ),
+  ]);
   const res = await soClient.find<AgentActionSOAttributes>({
     type: AGENT_ACTION_SAVED_OBJECT_TYPE,
-    filter: `not ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at: * AND ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.created_at >= "${timestamp}"`,
+    filter,
   });
 
   return res.saved_objects.map(savedObjectToAgentAction);