From f4c1aa723bde44ad844b62fd939998baa77d8573 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 22 Jul 2024 12:19:24 -0700 Subject: [PATCH 01/32] Update schemas to include `output_id` for package policies --- x-pack/plugins/fleet/common/types/models/package_policy.ts | 1 + x-pack/plugins/fleet/server/saved_objects/index.ts | 1 + x-pack/plugins/fleet/server/types/models/package_policy.ts | 6 ++---- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 1b8a407ff8dd92..c3564c50e84268 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -81,6 +81,7 @@ export interface NewPackagePolicy { /** @deprecated */ policy_id?: string; policy_ids: string[]; + output_id?: string; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; vars?: PackagePolicyConfigRecord; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 4b76a952a0f3b4..1b9a0938d31100 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -481,6 +481,7 @@ export const getSavedObjectTypes = ( is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 7ad705c9fc8af0..87eed8e00e80c9 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -98,6 +98,7 @@ const PackagePolicyBaseSchema = { namespace: schema.maybe(PackagePolicyNamespaceSchema), policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + output_id: schema.maybe(schema.string()), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), package: schema.maybe( @@ -109,8 +110,6 @@ const PackagePolicyBaseSchema = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)), vars: schema.maybe(ConfigRecordSchema), overrides: schema.maybe( @@ -152,8 +151,6 @@ const CreatePackagePolicyProps = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -220,6 +217,7 @@ export const SimplifiedCreatePackagePolicyRequestBodySchema = SimplifiedPackagePolicyBaseSchema.extends({ policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + output_id: schema.maybe(schema.string()), force: schema.maybe(schema.boolean()), package: schema.object({ name: schema.string(), From 04821c36ab0f5464f3ae83b7558cfb04af15ff18 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 22 Jul 2024 12:58:22 -0700 Subject: [PATCH 02/32] Adjust create/edit package policy APIs to allow output_id and gate it behind enterprise license check --- .../simplified_package_policy_helper.ts | 6 +++ .../common/types/rest_spec/package_policy.ts | 1 + .../server/routes/package_policy/handlers.ts | 47 +++++++++++-------- .../routes/package_policy/utils/index.ts | 10 ++++ 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts index 74dad485f8d3ef..bea1222ef4103a 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts @@ -43,6 +43,7 @@ export interface SimplifiedPackagePolicy { id?: string; policy_id?: string; policy_ids: string[]; + output_id?: string; namespace: string; name: string; description?: string; @@ -142,6 +143,7 @@ export function simplifiedPackagePolicytoNewPackagePolicy( const { policy_id: policyId, policy_ids: policyIds, + output_id: outputId, namespace, name, description, @@ -156,6 +158,10 @@ export function simplifiedPackagePolicytoNewPackagePolicy( description ); + if (outputId) { + packagePolicy.output_id = outputId; + } + if (packagePolicy.package && options?.experimental_data_stream_features) { packagePolicy.package.experimental_data_stream_features = options.experimental_data_stream_features; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts index ff190f0a556cd5..8b74a3142fa973 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts @@ -63,6 +63,7 @@ export type PostDeletePackagePoliciesResponse = Array<{ package?: PackagePolicyPackage; policy_id?: string; policy_ids?: string[]; + output_id?: string; // Support generic errors statusCode?: number; body?: { diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 9c259207a2b95d..c85c2b063261db 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -64,6 +64,7 @@ import type { SimplifiedPackagePolicy } from '../../../common/services/simplifie import { canUseMultipleAgentPolicies, + canUseOutputPerIntegration, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -241,19 +242,22 @@ export const createPackagePolicyHandler: FleetRequestHandler< const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username); let wasPackageAlreadyInstalled = false; - if ('output_id' in newPolicy) { - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - delete newPolicy.output_id; - } const spaceId = fleetContext.spaceId; try { if (!newPolicy.policy_id && (!newPolicy.policy_ids || newPolicy.policy_ids.length === 0)) { throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); } - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); + const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = + canUseMultipleAgentPolicies(); if ((newPolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); + throw new PackagePolicyRequestError(canUseMultipleAgentPoliciesErrorMessage); + } + + const { canUseOutputPerIntegrationResult, errorMessage: outputPerIntegrationErrorMessage } = + canUseOutputPerIntegration(); + if ((newPolicy.policy_ids ?? []).length > 1 && !canUseOutputPerIntegrationResult) { + throw new PackagePolicyRequestError(outputPerIntegrationErrorMessage); } let newPackagePolicy: NewPackagePolicy; @@ -365,13 +369,26 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } + if ( + !request.body.policy_id && + (!request.body.policy_ids || request.body.policy_ids.length === 0) + ) { + throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); + } + + const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); + if ((request.body.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyRequestError(errorMessage); + } + + const { canUseOutputPerIntegrationResult, errorMessage: outputPerIntegrationErrorMessage } = + canUseOutputPerIntegration(); + if ((request.body.output_id ?? []).length > 1 && !canUseOutputPerIntegrationResult) { + throw new PackagePolicyRequestError(outputPerIntegrationErrorMessage); + } + try { const { force, package: pkg, ...body } = request.body; - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - if ('output_id' in body) { - delete body.output_id; - } - let newData: NewPackagePolicy; if ( @@ -417,14 +434,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< newData.overrides = overrides; } } - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } - - if (newData.policy_ids && newData.policy_ids.length === 0) { - throw new PackagePolicyRequestError('At least one agent policy id must be provided'); - } await renameAgentlessAgentPolicy(soClient, esClient, packagePolicy, newData.name); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index 5177f6c3483756..17f497747acb97 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -55,6 +55,7 @@ export function removeFieldsFromInputSchema( } const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; +const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; export function canUseMultipleAgentPolicies() { const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); @@ -68,6 +69,15 @@ export function canUseMultipleAgentPolicies() { }; } +export function canUseOutputPerIntegration() { + const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + + return { + canUseOutputPerIntegrationResult: hasEnterpriseLicence, + errorMessage: 'Output per integration is only available with an Enterprise license', + }; +} + /** * If an agentless agent policy is associated with the package policy, * it will rename the agentless agent policy of a package policy to keep it in sync with the package policy name. From d11b8d85d8b3973696e949ede8c8d80d5ca364ce Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 22 Jul 2024 13:15:33 -0700 Subject: [PATCH 03/32] Adjust API gating to include check for allowed output type based on the package (i.e. APM can only have ES output) --- .../server/routes/package_policy/handlers.ts | 22 ++++++++------ .../routes/package_policy/utils/index.ts | 29 ++++++++++++++++--- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index c85c2b063261db..3ef3bd1fdd5c34 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -64,7 +64,7 @@ import type { SimplifiedPackagePolicy } from '../../../common/services/simplifie import { canUseMultipleAgentPolicies, - canUseOutputPerIntegration, + canUseOutputForIntegration, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -254,10 +254,12 @@ export const createPackagePolicyHandler: FleetRequestHandler< throw new PackagePolicyRequestError(canUseMultipleAgentPoliciesErrorMessage); } - const { canUseOutputPerIntegrationResult, errorMessage: outputPerIntegrationErrorMessage } = - canUseOutputPerIntegration(); - if ((newPolicy.policy_ids ?? []).length > 1 && !canUseOutputPerIntegrationResult) { - throw new PackagePolicyRequestError(outputPerIntegrationErrorMessage); + if (newPolicy.output_id && pkg) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration(soClient, pkg.name, newPolicy.output_id); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); + } } let newPackagePolicy: NewPackagePolicy; @@ -381,10 +383,12 @@ export const updatePackagePolicyHandler: FleetRequestHandler< throw new PackagePolicyRequestError(errorMessage); } - const { canUseOutputPerIntegrationResult, errorMessage: outputPerIntegrationErrorMessage } = - canUseOutputPerIntegration(); - if ((request.body.output_id ?? []).length > 1 && !canUseOutputPerIntegrationResult) { - throw new PackagePolicyRequestError(outputPerIntegrationErrorMessage); + if (request.body.output_id && request.body.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration(soClient, request.body.package.name, request.body.output_id); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); + } } try { diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index 17f497747acb97..a95f37b7a1b01e 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -21,7 +21,8 @@ import type { PackagePolicyInput, } from '../../../types'; import { appContextService } from '../../../services'; -import { agentPolicyService, licenseService } from '../../../services'; +import { agentPolicyService, licenseService, outputService } from '../../../services'; +import { getAllowedOutputTypesForIntegration } from '../../../../common/services/output_helpers'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; @@ -69,12 +70,32 @@ export function canUseMultipleAgentPolicies() { }; } -export function canUseOutputPerIntegration() { +export async function canUseOutputForIntegration( + soClient: SavedObjectsClientContract, + packageName: string, + outputId: string +) { const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasEnterpriseLicence) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: 'Output per integration is only available with an Enterprise license', + }; + } + + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const output = await outputService.get(soClient, outputId); + + if (!allowedOutputTypes.includes(output.type)) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}".`, + }; + } return { - canUseOutputPerIntegrationResult: hasEnterpriseLicence, - errorMessage: 'Output per integration is only available with an Enterprise license', + canUseOutputForIntegrationResult: true, + errorMessage: null, }; } From 06aba09ff0fb5080a0188bfd45c78f06d6c8ec1a Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 22 Jul 2024 17:33:19 -0700 Subject: [PATCH 04/32] Compile output permissions correctly --- .../full_agent_policy.test.ts.snap | 168 ++++++++++++++++++ .../agent_policies/full_agent_policy.test.ts | 120 +++++++++++++ .../agent_policies/full_agent_policy.ts | 48 +++-- .../package_policies_to_agent_inputs.test.ts | 5 +- .../package_policies_to_agent_inputs.ts | 8 +- .../agent_policies/related_saved_objects.ts | 15 +- 6 files changed, 345 insertions(+), 19 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 86f8126c1c45ae..8518e43fdaf0d2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -1,5 +1,173 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`getFullAgentPolicy should return the right outputs and permissions when package policies use their own outputs 1`] = ` +Object { + "agent": Object { + "download": Object { + "sourceURI": "http://default-registry.co", + }, + "features": Object {}, + "monitoring": Object { + "enabled": false, + "logs": false, + "metrics": false, + }, + "protection": Object { + "enabled": false, + "signing_key": "", + "uninstall_token_hash": "", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "integration-output-policy", + "inputs": Array [ + Object { + "data_stream": Object { + "namespace": "policyspace", + }, + "id": "test-logs-package-policy-using-output", + "meta": Object { + "package": Object { + "name": "test_package", + "version": "0.0.0", + }, + }, + "name": "test-policy-1", + "package_policy_id": "package-policy-using-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "test-remote-id", + }, + Object { + "data_stream": Object { + "namespace": "defaultspace", + }, + "id": "test-logs-package-policy-no-output", + "meta": Object { + "package": Object { + "name": "system", + "version": "1.0.0", + }, + }, + "name": "test-policy-2", + "package_policy_id": "package-policy-no-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "data-output-id", + }, + ], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-no-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-defaultspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_elastic_agent_monitoring": Object { + "indices": Array [ + Object { + "names": Array [], + "privileges": Array [], + }, + ], + }, + }, + "test-remote-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-using-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-policyspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "hosts": Array [ + "http://es-data.co:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "default": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "test-remote-id": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "service_token": undefined, + "type": "remote_elasticsearch", + }, + }, + "revision": 1, + "secret_references": Array [], + "signed": Object { + "data": "", + "signature": "", + }, +} +`; + exports[`getFullAgentPolicy should support a different data output 1`] = ` Object { "agent": Object { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 5701a60b56d037..6a084b5dde5865 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -401,6 +401,126 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy?.outputs['test-remote-id']).toBeDefined(); }); + it('should return the right outputs and permissions when package policies use their own outputs', async () => { + mockedGetPackageInfo.mockResolvedValue({ + data_streams: [ + { + type: 'logs', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'logs', + dataset: 'elastic_agent.filebeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.filebeat', + }, + ], + } as PackageInfo); + mockAgentPolicy({ + id: 'integration-output-policy', + status: 'active', + package_policies: [ + { + id: 'package-policy-using-output', + name: 'test-policy-1', + namespace: 'policyspace', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + output_id: 'test-remote-id', + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + { + id: 'package-policy-no-output', + name: 'test-policy-2', + namespace: '', + enabled: true, + package: { name: 'system', version: '1.0.0', title: 'System' }, + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ], + is_managed: false, + namespace: 'defaultspace', + revision: 1, + name: 'Policy', + updated_at: '2020-01-01', + updated_by: 'qwerty', + is_protected: false, + data_output_id: 'data-output-id', + }); + + const agentPolicy = await getFullAgentPolicy( + savedObjectsClientMock.create(), + 'integration-output-policy' + ); + expect(agentPolicy).toMatchSnapshot(); + }); + it('should return the sourceURI from the agent policy', async () => { mockAgentPolicy({ namespace: 'default', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index efc3a732149d64..0a8bc074614070 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -196,16 +196,41 @@ export async function getFullAgentPolicy( fullAgentPolicy.namespaces = [agentPolicy.space_id]; } - const dataPermissions = - (await storedPackagePoliciesToAgentPermissions( + const packagePoliciesByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = []; + return acc; + }, + {} + ); + (agentPolicy.package_policies || []).forEach((packagePolicy) => { + if (packagePolicy.output_id) { + packagePoliciesByOutputId[packagePolicy.output_id].push(packagePolicy); + } else { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(dataOutput)].push(packagePolicy); + } + }); + + const dataPermissionsByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = {}; + return acc; + }, + {} + ); + for (const [outputId, packagePolicies] of Object.entries(packagePoliciesByOutputId)) { + const dataPermissions = await storedPackagePoliciesToAgentPermissions( packageInfoCache, agentPolicy.namespace, - agentPolicy.package_policies - )) || {}; - - dataPermissions._elastic_agent_checks = { - cluster: DEFAULT_CLUSTER_PERMISSIONS, - }; + packagePolicies + ); + dataPermissionsByOutputId[outputId] = { + _elastic_agent_checks: { + cluster: DEFAULT_CLUSTER_PERMISSIONS, + }, + ...(dataPermissions || {}), + }; + } const monitoringPermissions = await getMonitoringPermissions( soClient, @@ -233,8 +258,11 @@ export async function getFullAgentPolicy( Object.assign(permissions, monitoringPermissions); } - if (outputId === getOutputIdForAgentPolicy(dataOutput)) { - Object.assign(permissions, dataPermissions); + if ( + outputId === getOutputIdForAgentPolicy(dataOutput) || + packagePoliciesByOutputId[outputId].length > 0 + ) { + Object.assign(permissions, dataPermissionsByOutputId[outputId]); } outputPermissions[outputId] = permissions; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts index 40cda7583a3c15..9004d544f399db 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts @@ -222,6 +222,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { version: '0.0.0', }, inputs: [mockInput, mockInput2], + output_id: 'new-output', }, { ...mockPackagePolicy, @@ -243,7 +244,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-logs', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', @@ -270,7 +271,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-metrics', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index d7f0c70a0786b7..807312fe5e7cb2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -27,7 +27,7 @@ const isPolicyEnabled = (packagePolicy: PackagePolicy) => { export const storedPackagePolicyToAgentInputs = ( packagePolicy: PackagePolicy, packageInfo?: PackageInfo, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, addFields?: FullAgentPolicyAddFields ): FullAgentPolicyInput[] => { @@ -62,7 +62,7 @@ export const storedPackagePolicyToAgentInputs = ( data_stream: { namespace: packagePolicy?.namespace || agentPolicyNamespace || 'default', // custom namespace has precedence on agent policy's one }, - use_output: outputId, + use_output: packagePolicy.output_id || agentPolicyOutputId, package_policy_id: packagePolicy.id, ...getFullInputStreams(input), }; @@ -140,7 +140,7 @@ export const getFullInputStreams = ( export const storedPackagePoliciesToAgentInputs = async ( packagePolicies: PackagePolicy[], packageInfoCache: Map, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, globalDataTags?: GlobalDataTag[] ): Promise => { @@ -164,7 +164,7 @@ export const storedPackagePoliciesToAgentInputs = async ( ...storedPackagePolicyToAgentInputs( packagePolicy, packageInfo, - outputId, + agentPolicyOutputId, agentPolicyNamespace, addFields ) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts index 0108e9cd97721c..52dd34a757693b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts @@ -35,11 +35,20 @@ export async function fetchRelatedSavedObjects( const monitoringOutputId = agentPolicy.monitoring_output_id || defaultMonitoringOutputId || dataOutputId; + const outputIds = uniq([ + dataOutputId, + monitoringOutputId, + ...(agentPolicy.package_policies || []).reduce((acc: string[], packagePolicy) => { + if (packagePolicy.output_id) { + acc.push(packagePolicy.output_id); + } + return acc; + }, []), + ]); + const [outputs, { host: downloadSourceUri, proxy_id: downloadSourceProxyId }, fleetServerHosts] = await Promise.all([ - outputService.bulkGet(soClient, uniq([dataOutputId, monitoringOutputId]), { - ignoreNotFound: true, - }), + outputService.bulkGet(soClient, outputIds, { ignoreNotFound: true }), getSourceUriForAgentPolicy(soClient, agentPolicy), getFleetServerHostsForAgentPolicy(soClient, agentPolicy).catch((err) => { appContextService From 3cf6b7179af9ebf936f7e07c21203ac1d862ad23 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 22 Jul 2024 19:17:43 -0700 Subject: [PATCH 05/32] Fix update handler, add tests --- ...plified_package_policy_helper.test.ts.snap | 1 + .../simplified_package_policy_helper.test.ts | 1 + .../routes/package_policy/handlers.test.ts | 105 +++++++++++++++++- .../server/routes/package_policy/handlers.ts | 37 +++--- .../routes/package_policy/utils/index.ts | 2 +- 5 files changed, 124 insertions(+), 22 deletions(-) diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap index 5d747ebd88ebfe..7c549b030a3378 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap @@ -228,6 +228,7 @@ Object { ], "name": "nginx-1", "namespace": "default", + "output_id": "output123", "package": Object { "name": "nginx", "title": "Nginx", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 0cde9d003dbb19..ff9d6eac7490fb 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -37,6 +37,7 @@ describe('toPackagePolicy', () => { namespace: 'default', policy_id: 'policy123', policy_ids: ['policy123'], + output_id: 'output123', description: 'Test description', inputs: { 'nginx-logfile': { diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index cd31cb59e82f17..2ce8c8f5396a90 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -16,6 +16,7 @@ import { agentPolicyService, appContextService, licenseService, + outputService, packagePolicyService, } from '../../services'; import { createAppContextStartContractMock, xpackMocks } from '../../mocks'; @@ -28,7 +29,6 @@ import type { AgentPolicy, FleetRequestHandler } from '../../types'; import type { PackagePolicy } from '../../types'; import { createPackagePolicyHandler, getPackagePoliciesHandler } from './handlers'; - import { registerRoutes } from '.'; const packagePolicyServiceMock = packagePolicyService as jest.Mocked; @@ -147,6 +147,19 @@ jest.mock('../../services/epm/packages', () => { }; }); +jest.mock('../../services/output', () => { + return { + outputService: { + get: jest.fn(() => + Promise.resolve({ + id: 'some-output', + type: 'elasticsearch', + }) + ), + }, + }; +}); + describe('When calling package policy', () => { let routerMock: jest.Mocked; let routeHandler: FleetRequestHandler; @@ -202,6 +215,7 @@ describe('When calling package policy', () => { }; beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); // @ts-ignore const postMock = routerMock.versioned.post.mock; // @ts-ignore @@ -244,6 +258,53 @@ describe('When calling package policy', () => { }, }); }); + + it('should throw if no enterprise license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + const request = getCreateKibanaRequest({ + ...newPolicy, + policy_id: '1', + output_id: 'some-output', + } as any); + await createPackagePolicyHandler(context, request as any, response); + expect(response.customError).toHaveBeenCalledWith({ + statusCode: 400, + body: { + message: 'Output per integration is only available with an Enterprise license', + }, + }); + }); + + it('should throw if enterprise license and an incompatible output_id for the package is given', async () => { + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const request = getCreateKibanaRequest({ + ...newPolicy, + policy_id: '1', + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + output_id: 'non-es-output', + } as any); + await createPackagePolicyHandler(context, request as any, response); + expect(response.customError).toHaveBeenCalledWith({ + statusCode: 400, + body: { + message: 'Output type "kafka" is not usable with package "apm"', + }, + }); + }); + + it('should not throw if enterprise license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const request = getCreateKibanaRequest({ + ...newPolicy, + policy_id: '1', + output_id: 'some-output', + } as any); + await createPackagePolicyHandler(context, request as any, response); + expect(response.customError).not.toHaveBeenCalledWith(); + }); }); describe('update api handler', () => { @@ -308,6 +369,7 @@ describe('When calling package policy', () => { }); beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) => Promise.resolve(newData as PackagePolicy) ); @@ -445,6 +507,47 @@ describe('When calling package policy', () => { }); }); + it('should throw if no enterprise license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + const request = getUpdateKibanaRequest({ + output_id: 'some-output', + } as any); + await routeHandler(context, request, response); + expect(response.customError).toHaveBeenCalledWith({ + statusCode: 400, + body: { + message: 'Output per integration is only available with an Enterprise license', + }, + }); + }); + + it('should throw if enterprise license and an incompatible output_id for the package is given', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + const request = getUpdateKibanaRequest({ + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + output_id: 'non-es-output', + } as any); + await routeHandler(context, request as any, response); + expect(response.customError).toHaveBeenCalledWith({ + statusCode: 400, + body: { + message: 'Output type "kafka" is not usable with package "apm"', + }, + }); + }); + + it('should not throw if enterprise license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + const request = getUpdateKibanaRequest({ + output_id: 'some-output', + } as any); + await routeHandler(context, request as any, response); + expect(response.customError).not.toHaveBeenCalledWith(); + }); + it('should rename the agentless agent policy to sync with the package policy name if agentless is enabled', async () => { jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any); jest diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 3ef3bd1fdd5c34..e101b5bffeaf96 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -371,26 +371,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } - if ( - !request.body.policy_id && - (!request.body.policy_ids || request.body.policy_ids.length === 0) - ) { - throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); - } - - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((request.body.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } - - if (request.body.output_id && request.body.package) { - const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = - await canUseOutputForIntegration(soClient, request.body.package.name, request.body.output_id); - if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { - throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); - } - } - try { const { force, package: pkg, ...body } = request.body; let newData: NewPackagePolicy; @@ -439,6 +419,23 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } + const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); + if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyRequestError(errorMessage); + } + + if (newData.policy_ids && newData.policy_ids.length === 0) { + throw new PackagePolicyRequestError('At least one agent policy id must be provided'); + } + + if (newData.output_id && newData.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration(soClient, newData.package.name, newData.output_id); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); + } + } + await renameAgentlessAgentPolicy(soClient, esClient, packagePolicy, newData.name); const updatedPackagePolicy = await packagePolicyService.update( diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index a95f37b7a1b01e..ce3c6a81952eda 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -89,7 +89,7 @@ export async function canUseOutputForIntegration( if (!allowedOutputTypes.includes(output.type)) { return { canUseOutputForIntegrationResult: false, - errorMessage: `Output type "${output.type}" is not usable with package "${packageName}".`, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, }; } From 4358b9bea2b8142942658f3e1f3797a6adb0edcd Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 23 Jul 2024 15:30:29 -0700 Subject: [PATCH 06/32] Remove output from package policies when it is deleted Include package policies in search for agent policies to be bumped when output is updated --- .../common/types/models/package_policy.ts | 3 +- .../server/mocks/package_policy.mocks.ts | 1 + .../agent_policies/output_helpers.test.ts | 46 ++++++++--- .../agent_policies/outputs_helpers.ts | 17 ++-- .../server/services/agent_policy.test.ts | 79 +++++++++++++------ .../fleet/server/services/agent_policy.ts | 48 ++++++++++- .../fleet/server/services/output.test.ts | 5 ++ .../plugins/fleet/server/services/output.ts | 7 ++ .../services/package_policies/utils.test.ts | 1 + .../server/services/package_policies/utils.ts | 2 + .../server/services/package_policy.test.ts | 55 +++++++++++++ .../fleet/server/services/package_policy.ts | 69 +++++++++++++++- .../server/services/package_policy_service.ts | 12 +++ .../fleet/server/types/so_attributes.ts | 2 + 14 files changed, 302 insertions(+), 45 deletions(-) diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index c3564c50e84268..eb054ade02114a 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -81,7 +81,8 @@ export interface NewPackagePolicy { /** @deprecated */ policy_id?: string; policy_ids: string[]; - output_id?: string; + // Nullable to allow user to reset to default outputs + output_id?: string | null; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; vars?: PackagePolicyConfigRecord; diff --git a/x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts b/x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts index 6f79a20e6d1f33..b98dafa9abfd05 100644 --- a/x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts +++ b/x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts @@ -28,6 +28,7 @@ const generatePackagePolicySOAttributesMock = ( updated_by: 'user-a', policy_id: '444-555-666', policy_ids: ['444-555-666'], + output_id: 'output-123', enabled: true, inputs: [], namespace: 'default', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index dfdcf26adaa4ed..3c921b71559993 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -13,7 +13,7 @@ import { appContextService } from '..'; import { outputService } from '../output'; import { validateOutputForPolicy } from '.'; -import { validateOutputForNewPackagePolicy } from './outputs_helpers'; +import { validateOutputForPackagePolicy } from './outputs_helpers'; jest.mock('../app_context'); jest.mock('../output'); @@ -254,14 +254,14 @@ describe('validateOutputForPolicy', () => { }); }); -describe('validateOutputForNewPackagePolicy', () => { - it('should not allow fleet_server integration to be added to a policy using a logstash output', async () => { +describe('validateOutputForPackagePolicy', () => { + it('should not allow fleet_server integration to be added or edited to a policy using a logstash output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'logstash', } as any); await expect( - validateOutputForNewPackagePolicy( + validateOutputForPackagePolicy( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -273,15 +273,29 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "fleet_server" cannot be added to agent policy "Agent policy" because it uses output type "logstash".' ); + await expect( + validateOutputForPackagePolicy( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'fleet_server', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "logstash" which cannot be used for integration "fleet_server".' + ); }); - it('should not allow apm integration to be added to a policy using a kafka output', async () => { + it('should not allow apm integration to be added or edited to a policy using a kafka output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'kafka', } as any); await expect( - validateOutputForNewPackagePolicy( + validateOutputForPackagePolicy( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -293,6 +307,20 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "apm" cannot be added to agent policy "Agent policy" because it uses output type "kafka".' ); + await expect( + validateOutputForPackagePolicy( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'apm', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "kafka" which cannot be used for integration "apm".' + ); }); it('should not allow synthetics integration to be added to a policy using a default logstash output', async () => { @@ -302,7 +330,7 @@ describe('validateOutputForNewPackagePolicy', () => { } as any); mockedOutputService.getDefaultDataOutputId.mockResolvedValue('default'); await expect( - validateOutputForNewPackagePolicy( + validateOutputForPackagePolicy( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -320,7 +348,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'logstash', } as any); - await validateOutputForNewPackagePolicy( + await validateOutputForPackagePolicy( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -335,7 +363,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'elasticsearch', } as any); - await validateOutputForNewPackagePolicy( + await validateOutputForPackagePolicy( savedObjectsClientMock.create(), { name: 'Agent policy', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index 67f5a7772aa52a..44f35cce33c1b1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -96,10 +96,11 @@ export async function validateOutputForPolicy( } } -export async function validateOutputForNewPackagePolicy( +export async function validateOutputForPackagePolicy( soClient: SavedObjectsClientContract, agentPolicy: AgentPolicy, - packageName: string + packageName: string, + isNewPackagePolicy: boolean = true ) { const allowedOutputTypeForPolicy = getAllowedOutputTypesForIntegration(packageName); @@ -109,9 +110,15 @@ export async function validateOutputForNewPackagePolicy( if (isOutputTypeRestricted) { const dataOutput = await getDataOutputForAgentPolicy(soClient, agentPolicy); if (!allowedOutputTypeForPolicy.includes(dataOutput.type)) { - throw new OutputInvalidError( - `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` - ); + if (isNewPackagePolicy) { + throw new OutputInvalidError( + `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` + ); + } else { + throw new OutputInvalidError( + `Agent policy "${agentPolicy.name}" uses output type "${dataOutput.type}" which cannot be used for integration "${packageName}".` + ); + } } } } diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index fe715b8aa57c93..9237891e9ca566 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -23,7 +23,7 @@ import type { NewAgentPolicy, PreconfiguredAgentPolicy, } from '../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { AGENT_POLICY_INDEX, SO_SEARCH_LIMIT } from '../../common'; @@ -43,32 +43,63 @@ import type { UninstallTokenServiceInterface } from './security/uninstall_token_ function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); + const mockPolicy = { + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes: agentPolicyAttributes as AgentPolicy, + }; mock.get.mockImplementation(async (type: string, id: string) => { return { - type, id, - references: [], - attributes: agentPolicyAttributes as AgentPolicy, + ...mockPolicy, }; }); - mock.find.mockImplementation(async (options) => { + mock.bulkGet.mockImplementation(async (options) => { return { - saved_objects: [ - { - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - attributes: { - fleet_server_hosts: ['http://fleetserver:8220'], - }, - type: 'ingest_manager_settings', - score: 1, - references: [], - }, - ], - total: 1, - page: 1, - per_page: 1, + saved_objects: [], }; }); + mock.find.mockImplementation(async (options) => { + switch (options.type) { + case AGENT_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [ + { + id: 'agent-policy-id', + score: 1, + ...mockPolicy, + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + case PACKAGE_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [], + total: 0, + page: 1, + per_page: 1, + }; + default: + return { + saved_objects: [ + { + id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', + attributes: { + fleet_server_hosts: ['http://fleetserver:8220'], + }, + type: 'ingest_manager_settings', + score: 1, + references: [], + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + } + }); return mock; } @@ -754,12 +785,12 @@ describe('Agent policy', () => { [ expect.objectContaining({ attributes: expect.objectContaining({ - fleet_server_hosts: ['http://fleetserver:8220'], - revision: NaN, - updated_by: 'system', + monitoring_enabled: ['metrics'], }), - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - type: 'ingest_manager_settings', + id: 'agent-policy-id', + namespace: undefined, + type: 'ingest-agent-policies', + version: undefined, }), ], expect.objectContaining({ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 94983e4929883a..1e8afb4ec2c9e1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -16,7 +16,7 @@ import type { SavedObjectsBulkUpdateObject, SavedObjectsBulkUpdateResponse, SavedObjectsClientContract, - SavedObjectsFindResult, + SavedObject, SavedObjectsUpdateResponse, } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; @@ -41,6 +41,7 @@ import { import type { HTTPAuthorizationHeader } from '../../common/http_authorization_header'; import { + PACKAGE_POLICY_SAVED_OBJECT_TYPE, AGENT_POLICY_SAVED_OBJECT_TYPE, AGENTS_PREFIX, FLEET_AGENT_POLICIES_SCHEMA_VERSION, @@ -56,6 +57,7 @@ import type { NewAgentPolicy, NewPackagePolicy, PackagePolicy, + PackagePolicySOAttributes, PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, @@ -866,7 +868,7 @@ class AgentPolicyService { private async _bumpPolicies( internalSoClientWithoutSpaceExtension: SavedObjectsClientContract, esClient: ElasticsearchClient, - savedObjectsResults: Array>, + savedObjectsResults: Array>, options?: { user?: AuthenticatedUser } ): Promise> { const bumpedPolicies = savedObjectsResults.map( @@ -931,7 +933,8 @@ class AgentPolicyService { const internalSoClientWithoutSpaceExtension = appContextService.getInternalUserSOClientWithoutSpaceExtension(); - const currentPolicies = + // All agent policies directly using output + const agentPoliciesUsingOutput = await internalSoClientWithoutSpaceExtension.find({ type: SAVED_OBJECT_TYPE, fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], @@ -940,10 +943,47 @@ class AgentPolicyService { perPage: SO_SEARCH_LIMIT, namespaces: ['*'], }); + + // All package policies directly using output + const packagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.find({ + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + fields: ['revision', 'output_id', 'namespaces'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + namespaces: ['*'], + }); + + const agentPolicyIdsDirectlyUsingOutput = agentPoliciesUsingOutput.saved_objects.map( + (agentPolicySO) => agentPolicySO.id + ); + const agentPolicyIdsOfPackagePoliciesUsingOutput = + packagePoliciesUsingOutput.saved_objects.reduce((acc: Set, packagePolicySO) => { + const newIds = packagePolicySO.attributes.policy_ids.filter((policyId) => { + return !agentPolicyIdsDirectlyUsingOutput.includes(policyId); + }); + return new Set([...acc, ...newIds]); + }, new Set()); + + // Agent policies of the identified package policies, excluding ones already retrieved directly + const agentPoliciesOfPackagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.bulkGet( + [...agentPolicyIdsOfPackagePoliciesUsingOutput].map((id) => ({ + type: SAVED_OBJECT_TYPE, + id, + fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], + namespaces: ['*'], + })) + ); + return this._bumpPolicies( internalSoClientWithoutSpaceExtension, esClient, - currentPolicies.saved_objects, + [ + ...agentPoliciesUsingOutput.saved_objects, + ...agentPoliciesOfPackagePoliciesUsingOutput.saved_objects, + ], options ); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 3bc9003162f448..ff7122c9f48a56 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -19,10 +19,12 @@ import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { auditLoggingService } from './audit_logging'; jest.mock('./app_context'); jest.mock('./agent_policy'); +jest.mock('./package_policy'); jest.mock('./audit_logging'); const mockedAuditLoggingService = auditLoggingService as jest.Mocked; @@ -43,6 +45,7 @@ mockedAppContextService.getLogger.mockImplementation(() => { mockedAppContextService.getExperimentalFeatures.mockReturnValue({} as any); const mockedAgentPolicyService = agentPolicyService as jest.Mocked; +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; const CLOUD_ID = 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw=='; @@ -293,6 +296,7 @@ describe('Output Service', () => { mockedAgentPolicyService.hasFleetServerIntegration.mockClear(); mockedAgentPolicyService.hasSyntheticsIntegration.mockClear(); mockedAgentPolicyService.removeOutputFromAll.mockReset(); + mockedPackagePolicyService.removeOutputFromAll.mockReset(); mockedAppContextService.getInternalUserSOClient.mockReset(); mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); @@ -1758,6 +1762,7 @@ describe('Output Service', () => { fromPreconfiguration: true, }); expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled(); + expect(mockedPackagePolicyService.removeOutputFromAll).toBeCalled(); expect(soClient.delete).toBeCalled(); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f42867f0686bb9..815604560c8932 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -63,6 +63,7 @@ import { import type { OutputType } from '../types'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; import { auditLoggingService } from './audit_logging'; @@ -740,6 +741,12 @@ class OutputService { throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`); } + await packagePolicyService.removeOutputFromAll( + soClient, + appContextService.getInternalUserESClient(), + id + ); + await agentPolicyService.removeOutputFromAll( soClient, appContextService.getInternalUserESClient(), diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts index 9bf58298b09d35..a36a7db4046941 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts @@ -38,6 +38,7 @@ describe('Package Policy Utils', () => { }, policy_id: '444-555-666', policy_ids: ['444-555-666'], + output_id: 'output-123', revision: 1, secret_references: [], updated_at: '2024-01-25T15:21:13.389Z', diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index cf2d69c8874b96..7466258de94fc8 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -21,6 +21,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, // `package` is a reserved keyword package: packageInfo, inputs, @@ -45,6 +46,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, package: packageInfo, inputs, vars, diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index a62a5048ec567d..748beebe5c3ad8 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -5096,6 +5096,61 @@ describe('Package policy service', () => { ); }); }); + + describe('removeOutputFromAll', () => { + it('should update policies using deleted output', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const updateSpy = jest.spyOn(packagePolicyService, 'update'); + soClient.find.mockResolvedValue({ + saved_objects: [ + { + id: 'package-policy-1', + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }, + ], + } as any); + soClient.get.mockImplementation((type, id, options): any => { + if (id === 'package-policy-1') { + return Promise.resolve({ + id, + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }); + } + }); + + await packagePolicyService.removeOutputFromAll(soClient, esClient, 'output-id-123'); + + expect(updateSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'package-policy-1', + { + name: 'policy1', + enabled: true, + policy_id: 'agent-policy-1', + policy_ids: ['agent-policy-1'], + output_id: null, + inputs: [], + } + ); + }); + }); }); describe('getUpgradeDryRunDiff', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 8ee996c69e5233..d3602aca930289 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -130,7 +130,7 @@ import { isSecretStorageEnabled, } from './secrets'; import { getPackageAssetsMap } from './epm/packages/get'; -import { validateOutputForNewPackagePolicy } from './agent_policies/outputs_helpers'; +import { validateOutputForPackagePolicy } from './agent_policies/outputs_helpers'; import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; @@ -228,7 +228,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { agentPolicies.push(agentPolicy); if (agentPolicy && enrichedPackagePolicy.package?.name) { - await validateOutputForNewPackagePolicy( + await validateOutputForPackagePolicy( soClient, agentPolicy, enrichedPackagePolicy.package?.name @@ -1967,6 +1967,71 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } + public async removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ) { + const packagePolicies = ( + await soClient.find({ + type: SAVED_OBJECT_TYPE, + fields: ['name', 'enabled', 'policy_ids', 'inputs', 'output_id'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + }) + ).saved_objects.map(mapPackagePolicySavedObjectToPackagePolicy); + + if (packagePolicies.length > 0) { + const getPackagePolicyUpdate = (packagePolicy: PackagePolicy) => ({ + name: packagePolicy.name, + enabled: packagePolicy.enabled, + policy_ids: packagePolicy.policy_ids, + inputs: packagePolicy.inputs, + output_id: packagePolicy.output_id === outputId ? null : packagePolicy.output_id, + }); + + // Validate that the new cleared/default output is valid for the package policies + // (from each of the associated agent policies) before updating any of them + await pMap( + packagePolicies, + async (packagePolicy) => { + const existingPackagePolicy = await this.get(soClient, packagePolicy.id); + + if (!existingPackagePolicy) { + throw new PackagePolicyNotFoundError('Package policy not found'); + } + + for (const policyId of packagePolicy.policy_ids) { + if (packagePolicy.package?.name) { + const agentPolicy = await agentPolicyService.get(soClient, policyId, true); + if (agentPolicy) { + await validateOutputForPackagePolicy( + soClient, + agentPolicy, + packagePolicy.package.name, + false + ); + } + } + } + }, + { + concurrency: 50, + } + ); + await pMap( + packagePolicies, + (packagePolicy) => { + this.update(soClient, esClient, packagePolicy.id, getPackagePolicyUpdate(packagePolicy)); + }, + { + concurrency: 50, + } + ); + } + } + fetchAllItemIds( soClient: SavedObjectsClientContract, { perPage = 1000, kuery }: PackagePolicyClientFetchAllItemIdsOptions = {} diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 1462d451388b52..fed46872ab6cba 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -219,6 +219,18 @@ export interface PackagePolicyClient { experimentalDataStreamFeatures: ExperimentalDataStreamFeature[]; }>; + /** + * Remove an output from all package policies that are using it, and replace the output by the default ones. + * @param soClient + * @param esClient + * @param outputId + */ + removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ): Promise; + /** * Returns an `AsyncIterable` for retrieving all integration policy IDs * @param soClient diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 79640a0f90e129..ef39759b206f55 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -121,6 +121,8 @@ export interface PackagePolicySOAttributes { inputs: PackagePolicyInput[]; policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; updated_at: string; updated_by: string; description?: string; From a5fbc223fe4db3a551353b6d157a3eec65d40103 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 23 Jul 2024 16:21:28 -0700 Subject: [PATCH 07/32] Fix output preconfiguration test mock --- .../fleet/server/services/preconfiguration/outputs.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index f6feffd24df535..e08d3f12b3ce76 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -6,6 +6,7 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + import { appContextService } from '..'; import type { PreconfiguredOutput } from '../../../common/types'; @@ -60,6 +61,9 @@ describe('output preconfiguration', () => { per_page: 0, total: 0, }); + internalSoClientWithoutSpaceExtension.bulkGet.mockResolvedValue({ + saved_objects: [], + }); mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.delete.mockReset(); From bb776faf231655f6f0b80a716a0f5e16897d6d0b Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 23 Jul 2024 16:55:32 -0700 Subject: [PATCH 08/32] Fix truncation and wrapping issues with agent policy name + description link --- .../public/components/agent_policy_summary_line.tsx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/fleet/public/components/agent_policy_summary_line.tsx b/x-pack/plugins/fleet/public/components/agent_policy_summary_line.tsx index b055b51728a437..8bbccb033827d2 100644 --- a/x-pack/plugins/fleet/public/components/agent_policy_summary_line.tsx +++ b/x-pack/plugins/fleet/public/components/agent_policy_summary_line.tsx @@ -15,6 +15,7 @@ import type { AgentPolicy, Agent } from '../../common/types'; import { useLink } from '../hooks'; const MIN_WIDTH: CSSProperties = { minWidth: 0 }; const NO_WRAP_WHITE_SPACE: CSSProperties = { whiteSpace: 'nowrap' }; +const WRAP_WHITE_SPACE: CSSProperties = { whiteSpace: 'normal' }; export const AgentPolicySummaryLine = memo<{ policy: AgentPolicy; @@ -27,7 +28,7 @@ export const AgentPolicySummaryLine = memo<{ const revision = agent ? agent.policy_revision : policy.revision; return ( - + - + - + {withDescription && description && ( - + {description} From 9ce8a8c81a7e97bd1ed504ef64de653e88901173 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 23 Jul 2024 16:57:45 -0700 Subject: [PATCH 09/32] Add Output column to integration policies table --- .../package_policies_table.tsx | 197 +++++++++++++----- 1 file changed, 142 insertions(+), 55 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx index 2cecdf37f30f59..0dbf758c4d9be4 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx @@ -37,6 +37,8 @@ import { usePermissionCheck, useStartServices, useMultipleAgentPolicies, + useGetOutputs, + useDefaultOutput, } from '../../../../../hooks'; import { pkgKeyFromPackageInfo } from '../../../../../services'; @@ -106,6 +108,16 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ return packagePolicy.policy_ids.length || 0; }, []); + const { data: outputsData, isLoading: isOutputsLoading } = useGetOutputs(); + const { output: defaultOutputData } = useDefaultOutput(); + const outputNamesById = useMemo(() => { + const outputs = outputsData?.items ?? []; + return outputs.reduce>((acc, output) => { + acc[output.id] = output.name; + return acc; + }, {}); + }, [outputsData]); + const columns = useMemo( (): EuiInMemoryTableProps['columns'] => [ { @@ -115,62 +127,71 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.nameColumnTitle', { defaultMessage: 'Integration policy', }), + width: '35%', render: (value: string, packagePolicy: InMemoryPackagePolicy) => ( - - - - - {value} - - {packagePolicy.description ? ( - -   - - - - - ) : null} - - - {canUseMultipleAgentPolicies && - canReadAgentPolicies && - canReadIntegrationPolicies && - getSharedPoliciesNumber(packagePolicy) > 1 && ( - - - } + + + + + - - {' '} - - - + + {value} + + - )} + {canUseMultipleAgentPolicies && + canReadAgentPolicies && + canReadIntegrationPolicies && + getSharedPoliciesNumber(packagePolicy) > 1 && ( + + + } + > + + {' '} + + + + + )} + + + {packagePolicy.description && ( + + + {packagePolicy.description} + + + )} ), }, @@ -270,8 +291,14 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ ) : ( <> {agentPolicy.namespace} +   = ({ ); }, }, + { + field: 'output_id', + name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.outputColumnTitle', { + defaultMessage: 'Output', + }), + render: (outputId: InMemoryPackagePolicy['output_id']) => { + if (isOutputsLoading) { + return null; + } + if (outputId) { + return {outputNamesById[outputId] || outputId}; + } + if (agentPolicy.data_output_id) { + return ( + <> + + {outputNamesById[agentPolicy.data_output_id] || agentPolicy.data_output_id} + +   + + + ); + } + if (defaultOutputData) { + return ( + <> + + {outputNamesById[defaultOutputData.id] || defaultOutputData.id} + +   + + + ); + } + }, + }, { name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.actionsColumnTitle', { defaultMessage: 'Actions', }), + width: '70px', actions: [ { render: (packagePolicy: InMemoryPackagePolicy) => { @@ -311,8 +395,11 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ agentPolicy, canUseMultipleAgentPolicies, canReadAgentPolicies, - canWriteIntegrationPolicies, getSharedPoliciesNumber, + canWriteIntegrationPolicies, + isOutputsLoading, + defaultOutputData, + outputNamesById, ] ); From ccc4702ae40a84ad0727c1d3865b041784c3ad80 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 09:03:47 -0700 Subject: [PATCH 10/32] Make output_id nullable in api schemas --- x-pack/plugins/fleet/server/types/models/package_policy.ts | 4 ++-- x-pack/plugins/fleet/server/types/models/preconfiguration.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 87eed8e00e80c9..484ba7f987db79 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -98,7 +98,7 @@ const PackagePolicyBaseSchema = { namespace: schema.maybe(PackagePolicyNamespaceSchema), policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), - output_id: schema.maybe(schema.string()), + output_id: schema.nullable(schema.maybe(schema.string())), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), package: schema.maybe( @@ -183,6 +183,7 @@ export const SimplifiedPackagePolicyBaseSchema = schema.object({ name: schema.string(), description: schema.maybe(schema.string()), namespace: schema.maybe(schema.string()), + output_id: schema.nullable(schema.maybe(schema.string())), vars: schema.maybe(SimplifiedVarsSchema), inputs: schema.maybe( schema.recordOf( @@ -217,7 +218,6 @@ export const SimplifiedCreatePackagePolicyRequestBodySchema = SimplifiedPackagePolicyBaseSchema.extends({ policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), - output_id: schema.maybe(schema.string()), force: schema.maybe(schema.boolean()), package: schema.object({ name: schema.string(), diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 6216eb4bbd3269..79586b1885ed97 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -162,6 +162,7 @@ export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( }), description: schema.maybe(schema.string()), namespace: schema.maybe(PackagePolicyNamespaceSchema), + output_id: schema.nullable(schema.maybe(schema.string())), inputs: schema.maybe( schema.arrayOf( schema.object({ From 93bf7e582fa46aca8cb0fa8917bb593d59cc4491 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 09:05:41 -0700 Subject: [PATCH 11/32] Move license const --- x-pack/plugins/fleet/common/constants/output.ts | 1 + .../routes/package_policy/handlers.test.ts | 16 ++++++++-------- .../server/routes/package_policy/utils/index.ts | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 8ddb8ed4d2f064..fb01ba991d3d26 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -29,6 +29,7 @@ export const DEFAULT_OUTPUT: NewOutput = { export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output'; export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum'; +export const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; /** * Kafka constants diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index 2ce8c8f5396a90..8c19635882aa7a 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -259,7 +259,7 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and output_id is provided', async () => { + it('should throw if no valid license and output_id is provided', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const request = getCreateKibanaRequest({ ...newPolicy, @@ -270,12 +270,12 @@ describe('When calling package policy', () => { expect(response.customError).toHaveBeenCalledWith({ statusCode: 400, body: { - message: 'Output per integration is only available with an Enterprise license', + message: 'Output per integration is only available with an enterprise license', }, }); }); - it('should throw if enterprise license and an incompatible output_id for the package is given', async () => { + it('should throw if valid license and an incompatible output_id for the package is given', async () => { jest .spyOn(outputService, 'get') .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); @@ -295,7 +295,7 @@ describe('When calling package policy', () => { }); }); - it('should not throw if enterprise license and output_id is provided', async () => { + it('should not throw if valid license and output_id is provided', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); const request = getCreateKibanaRequest({ ...newPolicy, @@ -507,7 +507,7 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and output_id is provided', async () => { + it('should throw if no valid license and output_id is provided', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); const request = getUpdateKibanaRequest({ output_id: 'some-output', @@ -516,12 +516,12 @@ describe('When calling package policy', () => { expect(response.customError).toHaveBeenCalledWith({ statusCode: 400, body: { - message: 'Output per integration is only available with an Enterprise license', + message: 'Output per integration is only available with an enterprise license', }, }); }); - it('should throw if enterprise license and an incompatible output_id for the package is given', async () => { + it('should throw if valid license and an incompatible output_id for the package is given', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); jest .spyOn(outputService, 'get') @@ -539,7 +539,7 @@ describe('When calling package policy', () => { }); }); - it('should not throw if enterprise license and output_id is provided', async () => { + it('should not throw if valid license and output_id is provided', async () => { jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); const request = getUpdateKibanaRequest({ output_id: 'some-output', diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index ce3c6a81952eda..bb1bca76bbd2dc 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -22,6 +22,7 @@ import type { } from '../../../types'; import { appContextService } from '../../../services'; import { agentPolicyService, licenseService, outputService } from '../../../services'; +import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../common/constants'; import { getAllowedOutputTypesForIntegration } from '../../../../common/services/output_helpers'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; @@ -56,7 +57,6 @@ export function removeFieldsFromInputSchema( } const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; -const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; export function canUseMultipleAgentPolicies() { const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); @@ -75,11 +75,11 @@ export async function canUseOutputForIntegration( packageName: string, outputId: string ) { - const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); - if (!hasEnterpriseLicence) { + const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasAllowedLicense) { return { canUseOutputForIntegrationResult: false, - errorMessage: 'Output per integration is only available with an Enterprise license', + errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, }; } From d07184306373bc6b38f906050a8b86082f38963c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 09:06:58 -0700 Subject: [PATCH 12/32] Add Output select field to create/edit package policy form --- .../components/steps/components/hooks.tsx | 22 ++++++++ .../steps/step_define_package_policy.tsx | 56 ++++++++++++++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx index 55b8ac90c749c3..c63339d682bb71 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx @@ -8,6 +8,10 @@ import { useMemo } from 'react'; import { useHistory } from 'react-router-dom'; +import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../../../../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../../../../../../../common/services/output_helpers'; +import { useGetOutputs, useLicense } from '../../../../../../hooks'; + export function useDataStreamId() { const history = useHistory(); @@ -16,3 +20,21 @@ export function useDataStreamId() { return searchParams.get('datastreamId') ?? undefined; }, [history.location.search]); } + +export function useOutputs(packageName: string) { + const licenseService = useLicense(); + const canUseOutputPerIntegration = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + const { data: outputsData, isLoading } = useGetOutputs(); + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const allowedOutputs = useMemo(() => { + if (!outputsData || !canUseOutputPerIntegration) { + return []; + } + return outputsData.items.filter((output) => allowedOutputTypes.includes(output.type)); + }, [allowedOutputTypes, canUseOutputPerIntegration, outputsData]); + return { + isLoading, + canUseOutputPerIntegration, + allowedOutputs, + }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx index c312b48a2d290a..43c3f8092e9218 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx @@ -20,6 +20,7 @@ import { EuiLink, EuiCallOut, EuiSpacer, + EuiSelect, } from '@elastic/eui'; import styled from 'styled-components'; @@ -32,6 +33,7 @@ import { isAdvancedVar } from '../../services'; import type { PackagePolicyValidationResults } from '../../services'; import { PackagePolicyInputVarField } from './components'; +import { useOutputs } from './components/hooks'; // on smaller screens, fields should be displayed in one column const FormGroupResponsiveFields = styled(EuiDescribedFormGroup)` @@ -81,6 +83,14 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ }); } + // Outputs + const { + isLoading: isOutputsLoading, + canUseOutputPerIntegration, + allowedOutputs, + } = useOutputs(packageInfo.name); + + // Managed policy const isManaged = packagePolicy.is_managed; return validationResults ? ( @@ -245,6 +255,7 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ {isShowingAdvanced ? ( + {/* Namespace */} + + {/* Output */} + {canUseOutputPerIntegration && ( + + + } + helpText={ + + } + > + ({ + value: output.id, + text: output.name, + })), + ]} + value={packagePolicy.output_id || ''} + onChange={(e) => { + updatePackagePolicy({ + output_id: e.target.value.trim() || null, + }); + }} + /> + + + )} + + {/* Data retention settings info */} Date: Wed, 24 Jul 2024 10:11:18 -0700 Subject: [PATCH 13/32] Fix typos, clean up unused agent count code --- .../fleet/common/constants/mappings.ts | 3 ++- .../use_fleet_proxy_form.tsx | 2 +- .../use_fleet_server_host_form.test.tsx | 4 --- .../services/agent_and_policies_count.tsx | 25 ------------------- .../fleet/server/services/output.test.ts | 4 +-- .../plugins/fleet/server/services/output.ts | 2 +- 6 files changed, 6 insertions(+), 34 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts index f804dc872e0d18..6499da7f86cc9a 100644 --- a/x-pack/plugins/fleet/common/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -6,7 +6,7 @@ */ /** - * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo. + * ATTENTION: Mappings for Fleet are defined in the Elasticsearch repo. * * The following mappings declared here closely mirror them * But they are only used to perform validation on the endpoints using ListWithKuery @@ -54,6 +54,7 @@ export const PACKAGE_POLICIES_MAPPINGS = { is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx index 2a4250bd9707a5..2d8a5b4aa1952a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx @@ -33,7 +33,7 @@ const ConfirmTitle = () => ( const ConfirmDescription: React.FunctionComponent = ({}) => ( ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx index 8df533f0206b77..95a94848f47940 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx @@ -11,10 +11,6 @@ import { createFleetTestRendererMock } from '../../../../../../mock'; import { useFleetServerHostsForm } from './use_fleet_server_host_form'; -jest.mock('../../services/agent_and_policies_count', () => ({ - ...jest.requireActual('../../services/agent_and_policies_count'), - getAgentAndPolicyCount: () => ({ agentCount: 0, agentPolicyCount: 0 }), -})); jest.mock('../../hooks/use_confirm_modal', () => ({ ...jest.requireActual('../../hooks/use_confirm_modal'), useConfirmModal: () => ({ confirm: () => true }), diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 7e8cdac388cfe5..2e0171e42b6e72 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -43,28 +43,3 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { return { agentPolicyCount, agentCount }; } - -export async function getAgentAndPolicyCount() { - const agentPolicies = await sendGetAgentPolicies({ - perPage: 0, - }); - - if (agentPolicies.error) { - throw agentPolicies.error; - } - const agentPolicyCount = agentPolicies.data?.total ?? 0; - - const agents = await sendGetAgents({ - page: 1, - perPage: 0, // We only need the count here - showInactive: false, - }); - - if (agents.error) { - throw agents.error; - } - - const agentCount = agents.data?.total ?? 0; - - return { agentPolicyCount, agentCount }; -} diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index ff7122c9f48a56..e62547d8d5d0c1 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -1416,7 +1416,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new Elasticsearch output.' ); }); @@ -1432,7 +1432,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new Elasticsearch output.' ); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index 815604560c8932..d09bdef14d124f 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -200,7 +200,7 @@ function validateOutputNotUsedInPolicy( dataOutputType )} output cannot be used with ${integrationName} integration in ${ agentPolicy.name - }. Please create a new ElasticSearch output.` + }. Please create a new Elasticsearch output.` ); } } From cca9d2f79c5f86712824c64bdbf94e3085555edb Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 10:11:51 -0700 Subject: [PATCH 14/32] Fix agent policy and agent count in edit output confirm modal --- .../services/agent_and_policies_count.tsx | 40 +++++++++++++++---- .../fleet/server/services/agent_policy.ts | 5 ++- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 2e0171e42b6e72..8720ede4f04b8f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -5,17 +5,24 @@ * 2.0. */ -import { sendGetAgentPolicies, sendGetAgents } from '../../../hooks'; +import { sendGetAgentPolicies, sendGetPackagePolicies, sendGetAgents } from '../../../hooks'; import type { Output } from '../../../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../../constants'; +import { + AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../../../constants'; export async function getAgentAndPolicyCountForOutput(output: Output) { - let kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + let agentPolicyKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + const packagePolicyKuery = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${output.id}"`; + if (output.is_default) { - kuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; + agentPolicyKuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; } + const agentPolicies = await sendGetAgentPolicies({ - kuery, + kuery: agentPolicyKuery, page: 1, perPage: SO_SEARCH_LIMIT, }); @@ -23,7 +30,26 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { if (agentPolicies.error) { throw agentPolicies.error; } - const agentPolicyCount = agentPolicies.data?.items?.length ?? 0; + + const packagePolicies = await sendGetPackagePolicies({ + kuery: packagePolicyKuery, + page: 1, + perPage: SO_SEARCH_LIMIT, + }); + + if (packagePolicies.error) { + throw agentPolicies.error; + } + + const agentPolicyIds = (agentPolicies.data?.items || []).map((policy) => policy.id); + const agentPolicyIdsFromPackagePolicies = (packagePolicies.data?.items || []).reduce( + (acc: string[], packagePolicy) => { + return [...acc, ...(packagePolicy.policy_ids || [])]; + }, + [] + ); + const uniqueAgentPolicyIds = new Set([...agentPolicyIds, ...agentPolicyIdsFromPackagePolicies]); + const agentPolicyCount = uniqueAgentPolicyIds.size; let agentCount = 0; if (agentPolicyCount > 0) { @@ -31,7 +57,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { page: 1, perPage: 0, // We only need the count here showInactive: false, - kuery: agentPolicies.data?.items.map((policy) => `policy_id:"${policy.id}"`).join(' or '), + kuery: [...uniqueAgentPolicyIds].map((id) => `policy_id:"${id}"`).join(' or '), }); if (agents.error) { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 5339fc363d9d48..2f992758e3e846 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -940,6 +940,7 @@ class AgentPolicyService { outputId: string, options?: { user?: AuthenticatedUser } ): Promise> { + const { useSpaceAwareness } = appContextService.getExperimentalFeatures(); const internalSoClientWithoutSpaceExtension = appContextService.getInternalUserSOClientWithoutSpaceExtension(); @@ -958,7 +959,7 @@ class AgentPolicyService { const packagePoliciesUsingOutput = await internalSoClientWithoutSpaceExtension.find({ type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, - fields: ['revision', 'output_id', 'namespaces'], + fields: ['output_id', 'namespaces', 'policy_ids'], searchFields: ['output_id'], search: escapeSearchQueryPhrase(outputId), perPage: SO_SEARCH_LIMIT, @@ -983,7 +984,7 @@ class AgentPolicyService { type: SAVED_OBJECT_TYPE, id, fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], - namespaces: ['*'], + ...(useSpaceAwareness ? { namespaces: ['*'] } : {}), })) ); From e96483108a8aded192b65effcd859a77096c33a2 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 11:00:29 -0700 Subject: [PATCH 15/32] Fix output id not being set on creation w/ non-simplified request --- x-pack/plugins/fleet/server/services/package_policy.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index d3602aca930289..b88322f23ed5f8 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -1747,6 +1747,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }, policy_id: newPolicy.policy_id ?? agentPolicyId, policy_ids: newPolicy.policy_ids ?? [agentPolicyId], + output_id: newPolicy.output_id, inputs: newPolicy.inputs[0]?.streams ? newPolicy.inputs : inputs, vars: newPolicy.vars || newPP.vars, }; From 1a43c5c24e35918c4fa779dd11a292a0f629479c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 24 Jul 2024 19:48:18 -0700 Subject: [PATCH 16/32] Fix query for retrieving an output's related agent policies --- .../plugins/fleet/server/services/output.ts | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index d09bdef14d124f..f22179ce2a3cab 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -38,6 +38,7 @@ import type { } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE, @@ -130,27 +131,56 @@ async function getAgentPoliciesPerOutput( outputId?: string, isDefault?: boolean ) { - let kuery: string; + let agentPoliciesKuery: string; + const packagePoliciesKuery: string = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${outputId}"`; if (outputId) { if (isDefault) { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; } } else { if (isDefault) { - kuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { return; } } - const agentPolicySO = await agentPolicyService.list(soClient, { - kuery, + // Get agent policies directly using output + const directAgentPolicies = await agentPolicyService.list(soClient, { + kuery: agentPoliciesKuery, perPage: SO_SEARCH_LIMIT, withPackagePolicies: true, }); - return agentPolicySO?.items; + const directAgentPolicyIds = directAgentPolicies?.items.map((policy) => policy.id); + + // Get package policies using output and derive agent policies from that which + // are not already identfied above. The IDs cannot be used as part of the kuery + // above since the underlying saved object client .find() only filters on attributes + const packagePolicySOs = await packagePolicyService.list(soClient, { + kuery: packagePoliciesKuery, + perPage: SO_SEARCH_LIMIT, + }); + const agentPolicyIdsFromPackagePolicies = [ + ...new Set( + packagePolicySOs?.items.reduce((acc: string[], packagePolicy) => { + return [ + ...acc, + ...packagePolicy.policy_ids.filter((id) => !directAgentPolicyIds?.includes(id)), + ]; + }, []) + ), + ]; + const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( + soClient, + agentPolicyIdsFromPackagePolicies, + { + withPackagePolicies: true, + } + ); + + return [...directAgentPolicies.items, ...agentPoliciesFromPackagePolicies]; } async function validateLogstashOutputNotUsedInAPMPolicy( From 8c1f570c50a7b29ff1fe21cbadeb0212d3de757e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Fri, 26 Jul 2024 12:35:11 -0700 Subject: [PATCH 17/32] Fix test failures --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- x-pack/plugins/fleet/server/services/output.test.ts | 5 +++++ x-pack/plugins/fleet/server/services/package_policy.test.ts | 1 + .../apis/agent_policy/__snapshots__/agent_policy.snap | 1 + x-pack/test/fleet_api_integration/apis/outputs/crud.ts | 4 ++-- 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index 02ad163c439318..d0a680b2158a00 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -117,7 +117,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "90625b4a5ded9d4867358fcccc14a57c0454fcee", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", - "ingest-package-policies": "2c0f7c72d211bb7d3076ce2fc0bd368f9c16d274", + "ingest-package-policies": "fa29dd3aa6182d51366e90b54ea2a8bc581af001", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index e62547d8d5d0c1..32869bc31f9a3f 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -291,6 +291,7 @@ describe('Output Service', () => { } as unknown as ReturnType; beforeEach(() => { + mockedAgentPolicyService.getByIDs.mockResolvedValue([]); mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.hasFleetServerIntegration.mockClear(); @@ -303,6 +304,10 @@ describe('Output Service', () => { mockedAgentPolicyService.update.mockReset(); }); + afterEach(() => { + mockedAgentPolicyService.getByIDs.mockClear(); + }); + describe('create', () => { describe('elasticsearch output', () => { it('works with a predefined id', async () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 748beebe5c3ad8..9505edd3556cb3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -2921,6 +2921,7 @@ describe('Package policy service', () => { version: 'WzYyMzcsMV0=', name: 'my-cis_kubernetes_benchmark', namespace: 'default', + output_id: null, description: '', package: { name: 'cis_kubernetes_benchmark', diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap index 9112ad20ad8609..a4d255613133ef 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap @@ -30,6 +30,7 @@ Object { "enabled": true, "name": "system-1", "namespace": "default", + "output_id": null, "package": Object { "name": "system", "requires_root": true, diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index e635ad8eafae79..015f29bd0cdb72 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -411,7 +411,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -432,7 +432,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); From 5be583aab2ac0502beb6dd803a9c700064c5188c Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 09:51:13 -0700 Subject: [PATCH 18/32] Fix mocks --- .../fleet/server/services/preconfiguration/outputs.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index e08d3f12b3ce76..3088814c8f8a32 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -32,6 +32,9 @@ const mockedOutputService = outputService as jest.Mocked; jest.mock('../app_context', () => ({ appContextService: { getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + getExperimentalFeatures: () => ({ + useSpaceAwareness: true, + }), getLogger: () => new Proxy( {}, From 700e0b9ab84f13c01eacf3915e8735296a3ac1ef Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 10:43:23 -0700 Subject: [PATCH 19/32] Make outputs service search across all spaces when retrieving policy references --- .../fleet/server/services/output.test.ts | 1 + .../plugins/fleet/server/services/output.ts | 64 +++++++++---------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 32869bc31f9a3f..0951657fae6dd0 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -222,6 +222,7 @@ function getMockedSoClient( }); mockedAppContextService.getInternalUserSOClient.mockReturnValue(soClient); + mockedAppContextService.getInternalUserSOClientWithoutSpaceExtension.mockReturnValue(soClient); return soClient; } diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f22179ce2a3cab..2748ad78e765b9 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -126,11 +126,9 @@ function outputSavedObjectToOutput(so: SavedObject): Output }; } -async function getAgentPoliciesPerOutput( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { +async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); let agentPoliciesKuery: string; const packagePoliciesKuery: string = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${outputId}"`; if (outputId) { @@ -148,7 +146,7 @@ async function getAgentPoliciesPerOutput( } // Get agent policies directly using output - const directAgentPolicies = await agentPolicyService.list(soClient, { + const directAgentPolicies = await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { kuery: agentPoliciesKuery, perPage: SO_SEARCH_LIMIT, withPackagePolicies: true, @@ -158,7 +156,7 @@ async function getAgentPoliciesPerOutput( // Get package policies using output and derive agent policies from that which // are not already identfied above. The IDs cannot be used as part of the kuery // above since the underlying saved object client .find() only filters on attributes - const packagePolicySOs = await packagePolicyService.list(soClient, { + const packagePolicySOs = await packagePolicyService.list(internalSoClientWithoutSpaceExtension, { kuery: packagePoliciesKuery, perPage: SO_SEARCH_LIMIT, }); @@ -173,7 +171,7 @@ async function getAgentPoliciesPerOutput( ), ]; const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( - soClient, + internalSoClientWithoutSpaceExtension, agentPolicyIdsFromPackagePolicies, { withPackagePolicies: true, @@ -183,12 +181,8 @@ async function getAgentPoliciesPerOutput( return [...directAgentPolicies.items, ...agentPoliciesFromPackagePolicies]; } -async function validateLogstashOutputNotUsedInAPMPolicy( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - const agentPolicies = await getAgentPoliciesPerOutput(soClient, outputId, isDefault); +async function validateLogstashOutputNotUsedInAPMPolicy(outputId?: string, isDefault?: boolean) { + const agentPolicies = await getAgentPoliciesPerOutput(outputId, isDefault); // Validate no policy with APM use that policy if (agentPolicies) { @@ -200,16 +194,19 @@ async function validateLogstashOutputNotUsedInAPMPolicy( } } -async function findPoliciesWithFleetServerOrSynthetics( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { +async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + // find agent policies by outputId // otherwise query all the policies const agentPolicies = outputId - ? await getAgentPoliciesPerOutput(soClient, outputId, isDefault) - : (await agentPolicyService.list(soClient, { withPackagePolicies: true }))?.items; + ? await getAgentPoliciesPerOutput(outputId, isDefault) + : ( + await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + withPackagePolicies: true, + }) + )?.items; const policiesWithFleetServer = agentPolicies?.filter((policy) => agentPolicyService.hasFleetServerIntegration(policy)) || []; @@ -236,7 +233,6 @@ function validateOutputNotUsedInPolicy( } async function validateTypeChanges( - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, id: string, data: Nullable>, @@ -244,12 +240,14 @@ async function validateTypeChanges( defaultDataOutputId: string | null, fromPreconfiguration: boolean ) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); const mergedIsDefault = data.is_default ?? originalOutput.is_default; const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient, id, mergedIsDefault); + await findPoliciesWithFleetServerOrSynthetics(id, mergedIsDefault); if (data.type === outputType.Logstash || originalOutput.type === outputType.Logstash) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, id, mergedIsDefault); + await validateLogstashOutputNotUsedInAPMPolicy(id, mergedIsDefault); } // prevent changing an ES output to logstash or kafka if it's used by fleet server or synthetics policies if ( @@ -261,7 +259,7 @@ async function validateTypeChanges( validateOutputNotUsedInPolicy(policiesWithSynthetics, data.type, 'Synthetics'); } await updateAgentPoliciesDataOutputId( - soClient, + internalSoClientWithoutSpaceExtension, esClient, data, mergedIsDefault, @@ -305,7 +303,7 @@ class OutputService { return appContextService.getInternalUserSOClient(fakeRequest); } - private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { + private async _getDefaultDataOutputsSO() { const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -434,7 +432,7 @@ class OutputService { } public async getDefaultDataOutputId(soClient: SavedObjectsClientContract) { - const outputs = await this._getDefaultDataOutputsSO(soClient); + const outputs = await this._getDefaultDataOutputsSO(); if (!outputs.saved_objects.length) { return null; @@ -485,7 +483,7 @@ class OutputService { const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, undefined, data.is_default); + await validateLogstashOutputNotUsedInAPMPolicy(undefined, data.is_default); if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { throw new FleetEncryptedSavedObjectEncryptionKeyRequired( `${output.type} output needs encrypted saved object api key to be set` @@ -493,7 +491,7 @@ class OutputService { } } const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient); + await findPoliciesWithFleetServerOrSynthetics(); await updateAgentPoliciesDataOutputId( soClient, esClient, @@ -771,14 +769,17 @@ class OutputService { throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`); } + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + await packagePolicyService.removeOutputFromAll( - soClient, + internalSoClientWithoutSpaceExtension, appContextService.getInternalUserESClient(), id ); await agentPolicyService.removeOutputFromAll( - soClient, + internalSoClientWithoutSpaceExtension, appContextService.getInternalUserESClient(), id ); @@ -845,7 +846,6 @@ class OutputService { const mergedType = data.type ?? originalOutput.type; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); await validateTypeChanges( - soClient, esClient, id, updateData, From a52ed50f5a62572219b5f0b4049d9fe8457078cf Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 10:47:13 -0700 Subject: [PATCH 20/32] Update openapi spec --- x-pack/plugins/fleet/common/openapi/bundled.json | 3 +-- x-pack/plugins/fleet/common/openapi/bundled.yaml | 1 - .../common/openapi/components/schemas/new_package_policy.yaml | 1 - .../openapi/components/schemas/update_package_policy.yaml | 2 -- 4 files changed, 1 insertion(+), 6 deletions(-) diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index cf3ee35fca6df4..727ef4c30f4fdb 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7484,8 +7484,7 @@ "type": "string" }, "output_id": { - "type": "string", - "deprecated": true + "type": "string" }, "inputs": { "type": "array", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ad592b9ea18479..7a19615cfe1da3 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -4804,7 +4804,6 @@ components: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml index 29a09174385ec5..956f51a8016f54 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml @@ -22,7 +22,6 @@ properties: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml index 795d92ff1d4a7d..f06a2c9ea49f3a 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml @@ -23,8 +23,6 @@ properties: type: string output_id: type: string - description: Not supported output can be set at the agent policy level only - deprecated: true inputs: type: array items: From eef75923ae768af7775ba5c77e04f8384f782e0e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 12:54:11 -0700 Subject: [PATCH 21/32] Fix types --- x-pack/plugins/fleet/server/mocks/index.ts | 1 + .../plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 9e517ad928ba3c..9dfb920251e76e 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -182,6 +182,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked { - // @ts-expect-error output_id property does not exists anymore delete packagePolicyDoc.attributes.output_id; return packagePolicyDoc; From 79537572237285e319cb2fc8481322e92271e031 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 14:48:59 -0700 Subject: [PATCH 22/32] Add new version for package policy SO --- x-pack/plugins/fleet/server/saved_objects/index.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 1b9a0938d31100..f6fcae4de6505c 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -640,6 +640,16 @@ export const getSavedObjectTypes = ( }, ], }, + '14': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + output_id: { type: 'keyword' }, + }, + }, + ], + }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, From 7ca5a4f4a4f3e422b93dddcbca77e052cd15eb13 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 6 Aug 2024 14:49:16 -0700 Subject: [PATCH 23/32] Return promise in pMap --- x-pack/plugins/fleet/server/services/package_policy.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index e2aab5d5f4b8c7..5dac0cc2b09e52 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -2037,7 +2037,12 @@ class PackagePolicyClientImpl implements PackagePolicyClient { await pMap( packagePolicies, (packagePolicy) => { - this.update(soClient, esClient, packagePolicy.id, getPackagePolicyUpdate(packagePolicy)); + return this.update( + soClient, + esClient, + packagePolicy.id, + getPackagePolicyUpdate(packagePolicy) + ); }, { concurrency: 50, From 1cbc17d847333e6c18732a761e9f17c51402eca2 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:20:15 +0000 Subject: [PATCH 24/32] [CI] Auto-commit changed files from 'node scripts/check_mappings_update --fix' --- packages/kbn-check-mappings-update-cli/current_fields.json | 1 + packages/kbn-check-mappings-update-cli/current_mappings.json | 3 +++ 2 files changed, 4 insertions(+) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index 4cec00e68dc423..ac54cbe178ee04 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -628,6 +628,7 @@ "is_managed", "name", "namespace", + "output_id", "overrides", "package", "package.name", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 9242f775f70944..c237691894010a 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2093,6 +2093,9 @@ "namespace": { "type": "keyword" }, + "output_id": { + "type": "keyword" + }, "overrides": { "index": false, "type": "flattened" From 12023f0b577904d8bd439846f3d02ba4ab365b57 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 7 Aug 2024 10:15:33 -0700 Subject: [PATCH 25/32] Update SO hash --- .../ci_checks/saved_objects/check_registered_types.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index d0a680b2158a00..9205c57cb4f9d5 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -117,7 +117,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "90625b4a5ded9d4867358fcccc14a57c0454fcee", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", - "ingest-package-policies": "fa29dd3aa6182d51366e90b54ea2a8bc581af001", + "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", From 43f2eea720f5a0fd8197480e52757adce843bcfd Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 7 Aug 2024 13:57:04 -0700 Subject: [PATCH 26/32] Add api integration tests --- .../apis/outputs/crud.ts | 136 +++++++++++++++++- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 890ceac8bb2187..d34184a8c9c3e8 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -120,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { } }; - const createAgentPolicy = async (spaceId?: string): Promise => { + const createAgentPolicy = async ( + spaceId?: string, + dataOutputId?: string + ): Promise => { const { body: testPolicyRes } = await supertest .post(spaceId ? `/s/${spaceId}/api/fleet/agent_policies` : `/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') @@ -128,6 +131,7 @@ export default function (providerContext: FtrProviderContext) { name: `test ${uuidV4()}`, description: '', namespace: 'default', + ...(dataOutputId ? { data_output_id: dataOutputId } : {}), }) .expect(200); @@ -148,6 +152,47 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }; + const createPackagePolicy = async ( + agentPolicyIds: string[], + spaceId?: string, + outputId?: string + ): Promise => { + const { body: testPolicyRes } = await supertest + .post(spaceId ? `/s/${spaceId}/api/fleet/package_policies` : `/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `package ${uuidV4()}`, + description: '', + namespace: 'default', + policy_ids: agentPolicyIds, + enabled: true, + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + ...(outputId ? { output_id: outputId } : {}), + }) + .expect(200); + + return testPolicyRes; + }; + + const deletePackagePolicy = async (packagePolicyId: string, spaceId?: string) => { + await supertest + .post( + spaceId + ? `/s/${spaceId}/api/fleet/package_policies/delete` + : `/api/fleet/package_policies/delete` + ) + .send({ + packagePolicyId, + }) + .set('kbn-xsrf', 'xxxx') + .expect(200); + }; + const getAgentPolicy = async ( policyId: string, spaceId?: string @@ -765,10 +810,28 @@ export default function (providerContext: FtrProviderContext) { }); it('should bump all policies in all spaces if updating the default output', async () => { - const [policy1, policy2, policy3] = await Promise.all([ - createAgentPolicy(), + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies with default output under agent policies not using default output + // to ensure that those agent policies still get bumped + const [packagePolicy1, packagePolicy2] = await Promise.all([ + createPackagePolicy([policy2.item.id], undefined, defaultOutputId), + createPackagePolicy([policy4.item.id], TEST_SPACE_ID, defaultOutputId), ]); await supertest @@ -781,20 +844,83 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); - const [updatedPolicy1, updatedPolicy2, updatedPolicy3] = await Promise.all([ + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ getAgentPolicy(policy1.item.id), getAgentPolicy(policy2.item.id), getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 1); - expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 2); expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 1); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 2); + + // cleanup + await Promise.all([ + deleteAgentPolicy(policy1.item.id), + deleteAgentPolicy(policy2.item.id), + deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), + deletePackagePolicy(packagePolicy1.item.id), + deletePackagePolicy(packagePolicy2.item.id, TEST_SPACE_ID), + ]); + }); + + it('should bump all policies in all spaces if updating non-default output', async () => { + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ + createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), + createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies under agent policies using default output to ensure those + // agent policies still get bumped + const [packagePolicy1, packagePolicy2] = await Promise.all([ + createPackagePolicy([policy1.item.id], undefined, nonDefaultOutput.item.id), + createPackagePolicy([policy3.item.id], TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + await supertest + .put(`/api/fleet/outputs/${nonDefaultOutput.item.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Nondefault Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + }) + .expect(200); + + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ + getAgentPolicy(policy1.item.id), + getAgentPolicy(policy2.item.id), + getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + + expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 2); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 2); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 1); // cleanup await Promise.all([ deleteAgentPolicy(policy1.item.id), deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), + deletePackagePolicy(packagePolicy1.item.id), + deletePackagePolicy(packagePolicy2.item.id, TEST_SPACE_ID), ]); }); }); From f59c5ca58565b1350e79d0401760954b0dd9c022 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 7 Aug 2024 14:46:17 -0700 Subject: [PATCH 27/32] Fix default output ID --- .../server/services/agent_policies/full_agent_policy.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index 0a8bc074614070..0cfce9eb264f45 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -204,8 +204,13 @@ export async function getFullAgentPolicy( {} ); (agentPolicy.package_policies || []).forEach((packagePolicy) => { - if (packagePolicy.output_id) { - packagePoliciesByOutputId[packagePolicy.output_id].push(packagePolicy); + const packagePolicyDataOutput = packagePolicy.output_id + ? outputs.find((output) => output.id === packagePolicy.output_id) + : undefined; + if (packagePolicyDataOutput) { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(packagePolicyDataOutput)].push( + packagePolicy + ); } else { packagePoliciesByOutputId[getOutputIdForAgentPolicy(dataOutput)].push(packagePolicy); } From 476971f6bdd0f4ce1fc17847ea4aed31032ac3ed Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 12 Aug 2024 15:08:59 -0700 Subject: [PATCH 28/32] Remove unnecessary cleanup --- .../fleet_api_integration/apis/outputs/crud.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index d34184a8c9c3e8..27fc8014aee9d2 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -179,20 +179,6 @@ export default function (providerContext: FtrProviderContext) { return testPolicyRes; }; - const deletePackagePolicy = async (packagePolicyId: string, spaceId?: string) => { - await supertest - .post( - spaceId - ? `/s/${spaceId}/api/fleet/package_policies/delete` - : `/api/fleet/package_policies/delete` - ) - .send({ - packagePolicyId, - }) - .set('kbn-xsrf', 'xxxx') - .expect(200); - }; - const getAgentPolicy = async ( policyId: string, spaceId?: string @@ -861,8 +847,6 @@ export default function (providerContext: FtrProviderContext) { deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), - deletePackagePolicy(packagePolicy1.item.id), - deletePackagePolicy(packagePolicy2.item.id, TEST_SPACE_ID), ]); }); @@ -919,8 +903,6 @@ export default function (providerContext: FtrProviderContext) { deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), - deletePackagePolicy(packagePolicy1.item.id), - deletePackagePolicy(packagePolicy2.item.id, TEST_SPACE_ID), ]); }); }); From e2e7a99a29bc0442982a4449da3091942a26f0a6 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 12 Aug 2024 16:04:02 -0700 Subject: [PATCH 29/32] Move validation of multiple agent policies and outputs to package policy service level --- .../fleet/common/constants/package_policy.ts | 2 + .../hooks/use_multiple_agent_policies.ts | 3 +- x-pack/plugins/fleet/server/errors/index.ts | 2 + .../server/routes/package_policy/handlers.ts | 28 ------- .../routes/package_policy/utils/index.ts | 48 +----------- .../agent_policies/output_helpers.test.ts | 18 ++--- .../agent_policies/outputs_helpers.ts | 2 +- .../server/services/package_policies/utils.ts | 78 ++++++++++++++++++- .../fleet/server/services/package_policy.ts | 20 +++-- 9 files changed, 106 insertions(+), 95 deletions(-) diff --git a/x-pack/plugins/fleet/common/constants/package_policy.ts b/x-pack/plugins/fleet/common/constants/package_policy.ts index 1645d1a86efb72..00b41a8a29de2f 100644 --- a/x-pack/plugins/fleet/common/constants/package_policy.ts +++ b/x-pack/plugins/fleet/common/constants/package_policy.ts @@ -13,3 +13,5 @@ export const inputsFormat = { Simplified: 'simplified', Legacy: 'legacy', } as const; + +export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; diff --git a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts index 2adf814e8ffba8..b85b944c6ee00b 100644 --- a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts +++ b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts @@ -5,12 +5,11 @@ * 2.0. */ +import { LICENCE_FOR_MULTIPLE_AGENT_POLICIES } from '../../common/constants'; import { ExperimentalFeaturesService } from '../services'; import { useLicense } from './use_license'; -export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - export function useMultipleAgentPolicies() { const licenseService = useLicense(); const { enableReusableIntegrationPolicies } = ExperimentalFeaturesService.get(); diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index e2f8b883804841..80d8116baaaa33 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -65,6 +65,8 @@ export class PackagePolicyNameExistsError extends FleetError {} export class BundledPackageLocationNotFoundError extends FleetError {} export class PackagePolicyRequestError extends FleetError {} +export class PackagePolicyMultipleAgentPoliciesError extends FleetError {} +export class PackagePolicyOutputError extends FleetError {} export class EnrollmentKeyNameExistsError extends FleetError {} export class HostedAgentPolicyRestrictionRelatedError extends FleetError { diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 0119d9f7d5cbe9..31e38125dead5d 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -63,8 +63,6 @@ import { import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; import { - canUseMultipleAgentPolicies, - canUseOutputForIntegration, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -249,20 +247,6 @@ export const createPackagePolicyHandler: FleetRequestHandler< throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); } - const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = - canUseMultipleAgentPolicies(); - if ((newPolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(canUseMultipleAgentPoliciesErrorMessage); - } - - if (newPolicy.output_id && pkg) { - const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = - await canUseOutputForIntegration(soClient, pkg.name, newPolicy.output_id); - if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { - throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); - } - } - let newPackagePolicy: NewPackagePolicy; if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) { if (!pkg) { @@ -421,23 +405,11 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } newData.inputs = alignInputsAndStreams(newData.inputs); - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } if (newData.policy_ids && newData.policy_ids.length === 0) { throw new PackagePolicyRequestError('At least one agent policy id must be provided'); } - if (newData.output_id && newData.package) { - const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = - await canUseOutputForIntegration(soClient, newData.package.name, newData.output_id); - if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { - throw new PackagePolicyRequestError(outputForIntegrationErrorMessage); - } - } - await renameAgentlessAgentPolicy(soClient, esClient, packagePolicy, newData.name); const updatedPackagePolicy = await packagePolicyService.update( diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index d6aabdcd1959d1..032dab4a07acc7 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -21,10 +21,7 @@ import type { PackagePolicyInput, NewPackagePolicyInput, } from '../../../types'; -import { appContextService } from '../../../services'; -import { agentPolicyService, licenseService, outputService } from '../../../services'; -import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../common/constants'; -import { getAllowedOutputTypesForIntegration } from '../../../../common/services/output_helpers'; +import { agentPolicyService } from '../../../services'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; import type { NewPackagePolicyInputStream } from '../../../../common'; @@ -58,49 +55,6 @@ export function removeFieldsFromInputSchema( }); } -const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - -export function canUseMultipleAgentPolicies() { - const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); - const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); - - return { - canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, - errorMessage: !hasEnterpriseLicence - ? 'Reusable integration policies are only available with an Enterprise license' - : 'Reusable integration policies are not supported', - }; -} - -export async function canUseOutputForIntegration( - soClient: SavedObjectsClientContract, - packageName: string, - outputId: string -) { - const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); - if (!hasAllowedLicense) { - return { - canUseOutputForIntegrationResult: false, - errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, - }; - } - - const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); - const output = await outputService.get(soClient, outputId); - - if (!allowedOutputTypes.includes(output.type)) { - return { - canUseOutputForIntegrationResult: false, - errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, - }; - } - - return { - canUseOutputForIntegrationResult: true, - errorMessage: null, - }; -} - /** * If an agentless agent policy is associated with the package policy, * it will rename the agentless agent policy of a package policy to keep it in sync with the package policy name. diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index 3c921b71559993..0afef0170a99f1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -13,7 +13,7 @@ import { appContextService } from '..'; import { outputService } from '../output'; import { validateOutputForPolicy } from '.'; -import { validateOutputForPackagePolicy } from './outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './outputs_helpers'; jest.mock('../app_context'); jest.mock('../output'); @@ -254,14 +254,14 @@ describe('validateOutputForPolicy', () => { }); }); -describe('validateOutputForPackagePolicy', () => { +describe('validateAgentPolicyOutputForIntegration', () => { it('should not allow fleet_server integration to be added or edited to a policy using a logstash output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'logstash', } as any); await expect( - validateOutputForPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -274,7 +274,7 @@ describe('validateOutputForPackagePolicy', () => { 'Integration "fleet_server" cannot be added to agent policy "Agent policy" because it uses output type "logstash".' ); await expect( - validateOutputForPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -295,7 +295,7 @@ describe('validateOutputForPackagePolicy', () => { type: 'kafka', } as any); await expect( - validateOutputForPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -308,7 +308,7 @@ describe('validateOutputForPackagePolicy', () => { 'Integration "apm" cannot be added to agent policy "Agent policy" because it uses output type "kafka".' ); await expect( - validateOutputForPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -330,7 +330,7 @@ describe('validateOutputForPackagePolicy', () => { } as any); mockedOutputService.getDefaultDataOutputId.mockResolvedValue('default'); await expect( - validateOutputForPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -348,7 +348,7 @@ describe('validateOutputForPackagePolicy', () => { type: 'logstash', } as any); - await validateOutputForPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -363,7 +363,7 @@ describe('validateOutputForPackagePolicy', () => { type: 'elasticsearch', } as any); - await validateOutputForPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index 44f35cce33c1b1..66f45427601e6e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -96,7 +96,7 @@ export async function validateOutputForPolicy( } } -export async function validateOutputForPackagePolicy( +export async function validateAgentPolicyOutputForIntegration( soClient: SavedObjectsClientContract, agentPolicy: AgentPolicy, packageName: string, diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index 7466258de94fc8..61ff380fec559f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -6,8 +6,18 @@ */ import type { SavedObject } from '@kbn/core-saved-objects-common/src/server_types'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { PackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { + LICENCE_FOR_OUTPUT_PER_INTEGRATION, + LICENCE_FOR_MULTIPLE_AGENT_POLICIES, +} from '../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../common/services/output_helpers'; +import type { PackagePolicy, NewPackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { PackagePolicyMultipleAgentPoliciesError, PackagePolicyOutputError } from '../../errors'; +import { licenseService } from '../license'; +import { outputService } from '../output'; +import { appContextService } from '../app_context'; export const mapPackagePolicySavedObjectToPackagePolicy = ({ /* eslint-disable @typescript-eslint/naming-convention */ @@ -61,3 +71,69 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ created_by, }; }; + +export async function preflightCheckPackagePolicy( + soClient: SavedObjectsClientContract, + packagePolicy: PackagePolicy | NewPackagePolicy +) { + // If package policy has multiple agent policies, check if they can be used + const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = + canUseMultipleAgentPolicies(); + if ((packagePolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyMultipleAgentPoliciesError(canUseMultipleAgentPoliciesErrorMessage); + } + + // If package policy has an output_id, check if it can be used + if (packagePolicy.output_id && packagePolicy.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration( + soClient, + packagePolicy.package.name, + packagePolicy.output_id + ); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyOutputError(outputForIntegrationErrorMessage); + } + } +} + +export function canUseMultipleAgentPolicies() { + const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); + const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); + + return { + canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, + errorMessage: !hasEnterpriseLicence + ? 'Reusable integration policies are only available with an Enterprise license' + : 'Reusable integration policies are not supported', + }; +} + +export async function canUseOutputForIntegration( + soClient: SavedObjectsClientContract, + packageName: string, + outputId: string +) { + const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasAllowedLicense) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, + }; + } + + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const output = await outputService.get(soClient, outputId); + + if (!allowedOutputTypes.includes(output.type)) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, + }; + } + + return { + canUseOutputForIntegrationResult: true, + errorMessage: null, + }; +} diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index 5dac0cc2b09e52..3b5a46432120b3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -115,6 +115,7 @@ import { sendTelemetryEvents } from './upgrade_sender'; import { handleExperimentalDatastreamFeatureOptIn, mapPackagePolicySavedObjectToPackagePolicy, + preflightCheckPackagePolicy, } from './package_policies'; import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { @@ -131,7 +132,7 @@ import { isSecretStorageEnabled, } from './secrets'; import { getPackageAssetsMap } from './epm/packages/get'; -import { validateOutputForPackagePolicy } from './agent_policies/outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './agent_policies/outputs_helpers'; import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; @@ -212,6 +213,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.debug(`Creating new package policy`); this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); let enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', @@ -228,8 +230,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const agentPolicy = await agentPolicyService.get(soClient, policyId, true); agentPolicies.push(agentPolicy); - if (agentPolicy && enrichedPackagePolicy.package?.name) { - await validateOutputForPackagePolicy( + // If package policy did not set an output_id, see if the agent policy's output is compatible + if (!packagePolicy.output_id && agentPolicy && enrichedPackagePolicy.package?.name) { + await validateAgentPolicyOutputForIntegration( soClient, agentPolicy, enrichedPackagePolicy.package?.name @@ -408,14 +411,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!packagePolicy.id) { packagePolicy.id = SavedObjectsUtils.generateId(); } - - this.keepPolicyIdInSync(packagePolicy); - auditLoggingService.writeCustomSoAuditLog({ action: 'create', id: packagePolicy.id, savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, }); + + this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); } const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids)); @@ -471,6 +474,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const { pkgInfo, assetsMap } = packageInfoAndAsset; + validatePackagePolicyOrThrow(packagePolicy, pkgInfo); inputs = pkgInfo ? await _compilePackagePolicyInputs( @@ -822,6 +826,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { try { logger.debug(`Starting update of package policy ${id}`); + await preflightCheckPackagePolicy(soClient, packagePolicyUpdate); enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyUpdate', packagePolicyUpdate, @@ -1050,6 +1055,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const id = packagePolicyUpdate.id; this.keepPolicyIdInSync(packagePolicyUpdate); const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; + await preflightCheckPackagePolicy(soClient, packagePolicy); const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); if (!oldPackagePolicy) { throw new PackagePolicyNotFoundError('Package policy not found'); @@ -2020,7 +2026,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (packagePolicy.package?.name) { const agentPolicy = await agentPolicyService.get(soClient, policyId, true); if (agentPolicy) { - await validateOutputForPackagePolicy( + await validateAgentPolicyOutputForIntegration( soClient, agentPolicy, packagePolicy.package.name, From 560b078881adc823060e33c17ebab24fa41f30d8 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 12 Aug 2024 16:44:45 -0700 Subject: [PATCH 30/32] Move tests --- .../routes/package_policy/handlers.test.ts | 177 ------------------ .../services/package_policies/utils.test.ts | 97 +++++++++- 2 files changed, 96 insertions(+), 178 deletions(-) diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts index b47746d04cd01e..468828da32c068 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.test.ts @@ -16,7 +16,6 @@ import { agentPolicyService, appContextService, licenseService, - outputService, packagePolicyService, } from '../../services'; import { createAppContextStartContractMock, xpackMocks } from '../../mocks'; @@ -147,19 +146,6 @@ jest.mock('../../services/epm/packages', () => { }; }); -jest.mock('../../services/output', () => { - return { - outputService: { - get: jest.fn(() => - Promise.resolve({ - id: 'some-output', - type: 'elasticsearch', - }) - ), - }, - }; -}); - describe('When calling package policy', () => { let routerMock: jest.Mocked; let routeHandler: FleetRequestHandler; @@ -235,76 +221,6 @@ describe('When calling package policy', () => { }, }); }); - - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should throw if no valid license and output_id is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getCreateKibanaRequest({ - ...newPolicy, - policy_id: '1', - output_id: 'some-output', - } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Output per integration is only available with an enterprise license', - }, - }); - }); - - it('should throw if valid license and an incompatible output_id for the package is given', async () => { - jest - .spyOn(outputService, 'get') - .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ - ...newPolicy, - policy_id: '1', - package: { name: 'apm', version: '1.0.0', title: 'APM' }, - output_id: 'non-es-output', - } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Output type "kafka" is not usable with package "apm"', - }, - }); - }); - - it('should not throw if valid license and output_id is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ - ...newPolicy, - policy_id: '1', - output_id: 'some-output', - } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith(); - }); }); describe('Update api handler', () => { @@ -455,99 +371,6 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.ok).toHaveBeenCalled(); - }); - - it('should throw if enterprise license and feature flag is disabled and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are not supported', - }, - }); - }); - - it('should throw if empty policy_ids are provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: [] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'At least one agent policy id must be provided', - }, - }); - }); - - it('should throw if no valid license and output_id is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getUpdateKibanaRequest({ - output_id: 'some-output', - } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Output per integration is only available with an enterprise license', - }, - }); - }); - - it('should throw if valid license and an incompatible output_id for the package is given', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(outputService, 'get') - .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); - const request = getUpdateKibanaRequest({ - package: { name: 'apm', version: '1.0.0', title: 'APM' }, - output_id: 'non-es-output', - } as any); - await routeHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Output type "kafka" is not usable with package "apm"', - }, - }); - }); - - it('should not throw if valid license and output_id is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getUpdateKibanaRequest({ - output_id: 'some-output', - } as any); - await routeHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith(); - }); - it('should rename the agentless agent policy to sync with the package policy name if agentless is enabled', async () => { jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any); jest diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts index a36a7db4046941..311bfeb339f20f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts @@ -4,10 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import { PackagePolicyMocks } from '../../mocks'; -import { mapPackagePolicySavedObjectToPackagePolicy } from './utils'; +import { appContextService } from '../app_context'; +import { licenseService } from '../license'; +import { outputService } from '../output'; + +import { mapPackagePolicySavedObjectToPackagePolicy, preflightCheckPackagePolicy } from './utils'; describe('Package Policy Utils', () => { describe('mapPackagePolicySavedObjectToPackagePolicy()', () => { @@ -48,4 +53,94 @@ describe('Package Policy Utils', () => { }); }); }); + + describe('preflightCheckPackagePolicy', () => { + beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); + jest.spyOn(appContextService, 'getExperimentalFeatures').mockClear(); + }); + const soClient = savedObjectsClientMock.create(); + const testPolicy = { + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_ids: ['test'], + inputs: [], + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + }; + + it('should throw if no enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError( + 'Reusable integration policies are only available with an Enterprise license' + ); + }); + + it('should throw if enterprise license and multiple policy_ids is provided but no feature flag', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError('Reusable integration policies are not supported'); + }); + + it('should not throw if enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).resolves.not.toThrow(); + }); + + it('should throw if no valid license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, output_id: 'some-output' }) + ).rejects.toThrowError('Output per integration is only available with an enterprise license'); + }); + + it('should throw if valid license and an incompatible output_id for the package is given', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'non-es-output', + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + }) + ).rejects.toThrowError('Output type "kafka" is not usable with package "apm"'); + }); + + it('should not throw if valid license and valid output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'es-output', type: 'elasticsearch' } as any); + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'es-output', + }) + ).resolves.not.toThrow(); + }); + }); }); From 0802091b97c0cf5915a385ad77e515091246b3b6 Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Mon, 12 Aug 2024 16:56:19 -0700 Subject: [PATCH 31/32] Fix bad merge --- .../package_policies_table.tsx | 154 ++++++------------ 1 file changed, 51 insertions(+), 103 deletions(-) diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx index 9e33a3289bfefb..ed9805fb5f75a1 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/details_page/components/package_policies/package_policies_table.tsx @@ -129,105 +129,59 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ }), width: '35%', render: (value: string, packagePolicy: InMemoryPackagePolicy) => ( - - - - - + + + {value} + {packagePolicy.description ? ( + +   + + + + + ) : null} + + + {canUseMultipleAgentPolicies && + canReadAgentPolicies && + canReadIntegrationPolicies && + getSharedPoliciesNumber(packagePolicy) > 1 && ( + + } - {...(canReadIntegrationPolicies && !agentPolicy.supports_agentless - ? { - href: getHref('edit_integration', { - policyId: agentPolicy.id, - packagePolicyId: packagePolicy.id, - }), - } - : { disabled: true })} > - {value} - {packagePolicy.description ? ( - -   - - - - - ) : null} - + + {' '} + + + - {canUseMultipleAgentPolicies && - canReadAgentPolicies && - canReadIntegrationPolicies && - getSharedPoliciesNumber(packagePolicy) > 1 && ( - - - } - > - - {value} - - - - )} - {canUseMultipleAgentPolicies && - canReadAgentPolicies && - canReadIntegrationPolicies && - getSharedPoliciesNumber(packagePolicy) > 1 && ( - - - } - > - - {' '} - - - - - )} - - - {packagePolicy.description && ( - - - {packagePolicy.description} - - - )} + )} ), }, @@ -327,14 +281,8 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ ) : ( <> {agentPolicy.namespace} -   Date: Mon, 12 Aug 2024 19:25:11 -0700 Subject: [PATCH 32/32] Fix types --- x-pack/test/fleet_api_integration/apis/outputs/crud.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 27fc8014aee9d2..b860a774ba1220 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -815,7 +815,7 @@ export default function (providerContext: FtrProviderContext) { // Create package policies with default output under agent policies not using default output // to ensure that those agent policies still get bumped - const [packagePolicy1, packagePolicy2] = await Promise.all([ + await Promise.all([ createPackagePolicy([policy2.item.id], undefined, defaultOutputId), createPackagePolicy([policy4.item.id], TEST_SPACE_ID, defaultOutputId), ]); @@ -870,7 +870,7 @@ export default function (providerContext: FtrProviderContext) { // Create package policies under agent policies using default output to ensure those // agent policies still get bumped - const [packagePolicy1, packagePolicy2] = await Promise.all([ + await Promise.all([ createPackagePolicy([policy1.item.id], undefined, nonDefaultOutput.item.id), createPackagePolicy([policy3.item.id], TEST_SPACE_ID, nonDefaultOutput.item.id), ]);