From 826641505cfdeadcd37e6aa70aeae8d00a70f44d Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Tue, 13 Aug 2024 08:49:57 -0700 Subject: [PATCH] [UII] Support integration-level outputs (#189125) ## Summary Resolves #143905. This PR adds support for integration-level outputs. This means that different integrations within the same agent policy can now be configured to send data to different locations. This feature is gated behind `enterprise` level subscription. For each input, the agent policy will configure sending data to the following outputs in decreasing order of priority: 1. Output set specifically on the integration policy 2. Output set specifically on the integration's parent agent policy (including the case where an integration policy belongs to multiple agent policies) 3. Global default data output set via Fleet Settings Integration-level outputs will respect the same rules as agent policy-level outputs: - Certain integrations are disallowed from using certain output types, attempting to add them to each other via creation, updating, or "defaulting", will fail - `fleet-server`, `synthetics`, and `apm` can only use same-cluster Elasticsearch output - When an output is deleted, any integrations that were specifically using it will "clear" their output configuration and revert back to either `#2` or `#3` in the above list - When an output is edited, all agent policies across all spaces that use it will be bumped to a new revision, this includes: - Agent policies that have that output specifically set in their settings (existing behavior) - Agent policies that contain integrations which specifically has that output set (new behavior) - When a proxy is edited, the same new revision bump above will apply for any outputs using that proxy The final agent policy YAML that is generated will have: - `outputs` block that includes: - Data and monitoring outputs set at the agent policy level (existing behavior) - Any additional outputs set at the integration level, if they differ from the above - `outputs_permissions` block that includes permissions for each Elasticsearch output depending on which integrations and/or agent monitoring are assigned to it Integration policies table now includes `Output` column. If the output is defaulting to agent policy-level output, or global setting output, a tooltip is shown: image Configuring an integration-level output is done under Advanced options in the policy editor. Setting to the blank value will "clear" the output configuration. The list of available outputs is filtered by what outputs are available for that integration (see above): image An example of failure: ES output cannot be changed to Kafka while there is an integration image ## TODO - [x] Adjust side effects of editing/deleting output when policies use it across different spaces - [x] Add API integration tests - [x] Update OpenAPI spec - [x] Create doc issue ### Checklist Delete any items that are not applicable to this PR. - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../current_fields.json | 1 + .../current_mappings.json | 3 + .../check_registered_types.test.ts | 2 +- .../fleet/common/constants/mappings.ts | 3 +- .../plugins/fleet/common/constants/output.ts | 1 + .../fleet/common/constants/package_policy.ts | 2 + .../plugins/fleet/common/openapi/bundled.json | 3 +- .../plugins/fleet/common/openapi/bundled.yaml | 1 - .../schemas/new_package_policy.yaml | 1 - .../schemas/update_package_policy.yaml | 2 - ...plified_package_policy_helper.test.ts.snap | 1 + .../simplified_package_policy_helper.test.ts | 1 + .../simplified_package_policy_helper.ts | 6 + .../common/types/models/package_policy.ts | 2 + .../common/types/rest_spec/package_policy.ts | 1 + .../components/steps/components/hooks.tsx | 22 +++ .../steps/step_define_package_policy.tsx | 56 +++++- .../package_policies_table.tsx | 75 +++++++- .../use_fleet_proxy_form.tsx | 2 +- .../use_fleet_server_host_form.test.tsx | 4 - .../services/agent_and_policies_count.tsx | 65 +++---- .../hooks/use_multiple_agent_policies.ts | 3 +- x-pack/plugins/fleet/server/errors/index.ts | 2 + x-pack/plugins/fleet/server/mocks/index.ts | 1 + .../server/mocks/package_policy.mocks.ts | 1 + .../routes/package_policy/handlers.test.ts | 78 +------- .../server/routes/package_policy/handlers.ts | 19 -- .../routes/package_policy/utils/index.ts | 17 +- .../fleet/server/saved_objects/index.ts | 11 ++ .../saved_objects/migrations/to_v8_5_0.ts | 1 - .../full_agent_policy.test.ts.snap | 168 ++++++++++++++++++ .../agent_policies/full_agent_policy.test.ts | 120 +++++++++++++ .../agent_policies/full_agent_policy.ts | 53 ++++-- .../agent_policies/output_helpers.test.ts | 46 ++++- .../agent_policies/outputs_helpers.ts | 17 +- .../package_policies_to_agent_inputs.test.ts | 5 +- .../package_policies_to_agent_inputs.ts | 8 +- .../agent_policies/related_saved_objects.ts | 15 +- .../server/services/agent_policy.test.ts | 79 +++++--- .../fleet/server/services/agent_policy.ts | 49 ++++- .../fleet/server/services/output.test.ts | 15 +- .../plugins/fleet/server/services/output.ts | 109 ++++++++---- .../services/package_policies/utils.test.ts | 98 +++++++++- .../server/services/package_policies/utils.ts | 80 ++++++++- .../server/services/package_policy.test.ts | 56 ++++++ .../fleet/server/services/package_policy.ts | 89 +++++++++- .../server/services/package_policy_service.ts | 12 ++ .../services/preconfiguration/outputs.test.ts | 7 + .../server/types/models/package_policy.ts | 6 +- .../server/types/models/preconfiguration.ts | 1 + .../fleet/server/types/so_attributes.ts | 2 + .../__snapshots__/agent_policy.snap | 1 + .../apis/outputs/crud.ts | 122 ++++++++++++- 53 files changed, 1266 insertions(+), 279 deletions(-) diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index dd233e14648bd7..211777a5274a5b 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -630,6 +630,7 @@ "is_managed", "name", "namespace", + "output_id", "overrides", "package", "package.name", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 4ded89844052be..e6e1fef63ee85a 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2101,6 +2101,9 @@ "namespace": { "type": "keyword" }, + "output_id": { + "type": "keyword" + }, "overrides": { "index": false, "type": "flattened" diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index af58fd9d383bc6..6d978b2d33ca45 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -119,7 +119,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "90625b4a5ded9d4867358fcccc14a57c0454fcee", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", - "ingest-package-policies": "2c0f7c72d211bb7d3076ce2fc0bd368f9c16d274", + "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", diff --git a/x-pack/plugins/fleet/common/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts index f804dc872e0d18..6499da7f86cc9a 100644 --- a/x-pack/plugins/fleet/common/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -6,7 +6,7 @@ */ /** - * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo. + * ATTENTION: Mappings for Fleet are defined in the Elasticsearch repo. * * The following mappings declared here closely mirror them * But they are only used to perform validation on the endpoints using ListWithKuery @@ -54,6 +54,7 @@ export const PACKAGE_POLICIES_MAPPINGS = { is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 8ddb8ed4d2f064..fb01ba991d3d26 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -29,6 +29,7 @@ export const DEFAULT_OUTPUT: NewOutput = { export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output'; export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum'; +export const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; /** * Kafka constants diff --git a/x-pack/plugins/fleet/common/constants/package_policy.ts b/x-pack/plugins/fleet/common/constants/package_policy.ts index 1645d1a86efb72..00b41a8a29de2f 100644 --- a/x-pack/plugins/fleet/common/constants/package_policy.ts +++ b/x-pack/plugins/fleet/common/constants/package_policy.ts @@ -13,3 +13,5 @@ export const inputsFormat = { Simplified: 'simplified', Legacy: 'legacy', } as const; + +export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index cf3ee35fca6df4..727ef4c30f4fdb 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7484,8 +7484,7 @@ "type": "string" }, "output_id": { - "type": "string", - "deprecated": true + "type": "string" }, "inputs": { "type": "array", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ad592b9ea18479..7a19615cfe1da3 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -4804,7 +4804,6 @@ components: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml index 29a09174385ec5..956f51a8016f54 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml @@ -22,7 +22,6 @@ properties: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml index 795d92ff1d4a7d..f06a2c9ea49f3a 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml @@ -23,8 +23,6 @@ properties: type: string output_id: type: string - description: Not supported output can be set at the agent policy level only - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap index 5d747ebd88ebfe..7c549b030a3378 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap @@ -228,6 +228,7 @@ Object { ], "name": "nginx-1", "namespace": "default", + "output_id": "output123", "package": Object { "name": "nginx", "title": "Nginx", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 0cde9d003dbb19..ff9d6eac7490fb 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -37,6 +37,7 @@ describe('toPackagePolicy', () => { namespace: 'default', policy_id: 'policy123', policy_ids: ['policy123'], + output_id: 'output123', description: 'Test description', inputs: { 'nginx-logfile': { diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts index 98dee1021fe213..8d616bf4cae11a 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts @@ -43,6 +43,7 @@ export interface SimplifiedPackagePolicy { id?: string; policy_id?: string; policy_ids: string[]; + output_id?: string; namespace: string; name: string; description?: string; @@ -147,6 +148,7 @@ export function simplifiedPackagePolicytoNewPackagePolicy( const { policy_id: policyId, policy_ids: policyIds, + output_id: outputId, namespace, name, description, @@ -161,6 +163,10 @@ export function simplifiedPackagePolicytoNewPackagePolicy( description ); + if (outputId) { + packagePolicy.output_id = outputId; + } + if (packagePolicy.package && options?.experimental_data_stream_features) { packagePolicy.package.experimental_data_stream_features = options.experimental_data_stream_features; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 1b8a407ff8dd92..eb054ade02114a 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -81,6 +81,8 @@ export interface NewPackagePolicy { /** @deprecated */ policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; vars?: PackagePolicyConfigRecord; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts index ff190f0a556cd5..8b74a3142fa973 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts @@ -63,6 +63,7 @@ export type PostDeletePackagePoliciesResponse = Array<{ package?: PackagePolicyPackage; policy_id?: string; policy_ids?: string[]; + output_id?: string; // Support generic errors statusCode?: number; body?: { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx index 55b8ac90c749c3..c63339d682bb71 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx @@ -8,6 +8,10 @@ import { useMemo } from 'react'; import { useHistory } from 'react-router-dom'; +import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../../../../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../../../../../../../common/services/output_helpers'; +import { useGetOutputs, useLicense } from '../../../../../../hooks'; + export function useDataStreamId() { const history = useHistory(); @@ -16,3 +20,21 @@ export function useDataStreamId() { return searchParams.get('datastreamId') ?? undefined; }, [history.location.search]); } + +export function useOutputs(packageName: string) { + const licenseService = useLicense(); + const canUseOutputPerIntegration = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + const { data: outputsData, isLoading } = useGetOutputs(); + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const allowedOutputs = useMemo(() => { + if (!outputsData || !canUseOutputPerIntegration) { + return []; + } + return outputsData.items.filter((output) => allowedOutputTypes.includes(output.type)); + }, [allowedOutputTypes, canUseOutputPerIntegration, outputsData]); + return { + isLoading, + canUseOutputPerIntegration, + allowedOutputs, + }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx index c312b48a2d290a..43c3f8092e9218 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx @@ -20,6 +20,7 @@ import { EuiLink, EuiCallOut, EuiSpacer, + EuiSelect, } from '@elastic/eui'; import styled from 'styled-components'; @@ -32,6 +33,7 @@ import { isAdvancedVar } from '../../services'; import type { PackagePolicyValidationResults } from '../../services'; import { PackagePolicyInputVarField } from './components'; +import { useOutputs } from './components/hooks'; // on smaller screens, fields should be displayed in one column const FormGroupResponsiveFields = styled(EuiDescribedFormGroup)` @@ -81,6 +83,14 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ }); } + // Outputs + const { + isLoading: isOutputsLoading, + canUseOutputPerIntegration, + allowedOutputs, + } = useOutputs(packageInfo.name); + + // Managed policy const isManaged = packagePolicy.is_managed; return validationResults ? ( @@ -245,6 +255,7 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ {isShowingAdvanced ? ( + {/* Namespace */} + + {/* Output */} + {canUseOutputPerIntegration && ( + + + } + helpText={ + + } + > + ({ + value: output.id, + text: output.name, + })), + ]} + value={packagePolicy.output_id || ''} + onChange={(e) => { + updatePackagePolicy({ + output_id: e.target.value.trim() || null, + }); + }} + /> + + + )} + + {/* Data retention settings info */} = ({ return packagePolicy.policy_ids.length || 0; }, []); + const { data: outputsData, isLoading: isOutputsLoading } = useGetOutputs(); + const { output: defaultOutputData } = useDefaultOutput(); + const outputNamesById = useMemo(() => { + const outputs = outputsData?.items ?? []; + return outputs.reduce>((acc, output) => { + acc[output.id] = output.name; + return acc; + }, {}); + }, [outputsData]); + const columns = useMemo( (): EuiInMemoryTableProps['columns'] => [ { @@ -115,6 +127,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.nameColumnTitle', { defaultMessage: 'Integration policy', }), + width: '35%', render: (value: string, packagePolicy: InMemoryPackagePolicy) => ( @@ -278,10 +291,67 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ ); }, }, + { + field: 'output_id', + name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.outputColumnTitle', { + defaultMessage: 'Output', + }), + render: (outputId: InMemoryPackagePolicy['output_id']) => { + if (isOutputsLoading) { + return null; + } + if (outputId) { + return {outputNamesById[outputId] || outputId}; + } + if (agentPolicy.data_output_id) { + return ( + <> + + {outputNamesById[agentPolicy.data_output_id] || agentPolicy.data_output_id} + +   + + + ); + } + if (defaultOutputData) { + return ( + <> + + {outputNamesById[defaultOutputData.id] || defaultOutputData.id} + +   + + + ); + } + }, + }, { name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.actionsColumnTitle', { defaultMessage: 'Actions', }), + width: '70px', actions: [ { render: (packagePolicy: InMemoryPackagePolicy) => { @@ -309,8 +379,11 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ agentPolicy, canUseMultipleAgentPolicies, canReadAgentPolicies, - canWriteIntegrationPolicies, getSharedPoliciesNumber, + canWriteIntegrationPolicies, + isOutputsLoading, + defaultOutputData, + outputNamesById, ] ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx index 2a4250bd9707a5..2d8a5b4aa1952a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx @@ -33,7 +33,7 @@ const ConfirmTitle = () => ( const ConfirmDescription: React.FunctionComponent = ({}) => ( ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx index 8df533f0206b77..95a94848f47940 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx @@ -11,10 +11,6 @@ import { createFleetTestRendererMock } from '../../../../../../mock'; import { useFleetServerHostsForm } from './use_fleet_server_host_form'; -jest.mock('../../services/agent_and_policies_count', () => ({ - ...jest.requireActual('../../services/agent_and_policies_count'), - getAgentAndPolicyCount: () => ({ agentCount: 0, agentPolicyCount: 0 }), -})); jest.mock('../../hooks/use_confirm_modal', () => ({ ...jest.requireActual('../../hooks/use_confirm_modal'), useConfirmModal: () => ({ confirm: () => true }), diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 7e8cdac388cfe5..8720ede4f04b8f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -5,17 +5,24 @@ * 2.0. */ -import { sendGetAgentPolicies, sendGetAgents } from '../../../hooks'; +import { sendGetAgentPolicies, sendGetPackagePolicies, sendGetAgents } from '../../../hooks'; import type { Output } from '../../../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../../constants'; +import { + AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../../../constants'; export async function getAgentAndPolicyCountForOutput(output: Output) { - let kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + let agentPolicyKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + const packagePolicyKuery = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${output.id}"`; + if (output.is_default) { - kuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; + agentPolicyKuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; } + const agentPolicies = await sendGetAgentPolicies({ - kuery, + kuery: agentPolicyKuery, page: 1, perPage: SO_SEARCH_LIMIT, }); @@ -23,7 +30,26 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { if (agentPolicies.error) { throw agentPolicies.error; } - const agentPolicyCount = agentPolicies.data?.items?.length ?? 0; + + const packagePolicies = await sendGetPackagePolicies({ + kuery: packagePolicyKuery, + page: 1, + perPage: SO_SEARCH_LIMIT, + }); + + if (packagePolicies.error) { + throw agentPolicies.error; + } + + const agentPolicyIds = (agentPolicies.data?.items || []).map((policy) => policy.id); + const agentPolicyIdsFromPackagePolicies = (packagePolicies.data?.items || []).reduce( + (acc: string[], packagePolicy) => { + return [...acc, ...(packagePolicy.policy_ids || [])]; + }, + [] + ); + const uniqueAgentPolicyIds = new Set([...agentPolicyIds, ...agentPolicyIdsFromPackagePolicies]); + const agentPolicyCount = uniqueAgentPolicyIds.size; let agentCount = 0; if (agentPolicyCount > 0) { @@ -31,7 +57,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { page: 1, perPage: 0, // We only need the count here showInactive: false, - kuery: agentPolicies.data?.items.map((policy) => `policy_id:"${policy.id}"`).join(' or '), + kuery: [...uniqueAgentPolicyIds].map((id) => `policy_id:"${id}"`).join(' or '), }); if (agents.error) { @@ -43,28 +69,3 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { return { agentPolicyCount, agentCount }; } - -export async function getAgentAndPolicyCount() { - const agentPolicies = await sendGetAgentPolicies({ - perPage: 0, - }); - - if (agentPolicies.error) { - throw agentPolicies.error; - } - const agentPolicyCount = agentPolicies.data?.total ?? 0; - - const agents = await sendGetAgents({ - page: 1, - perPage: 0, // We only need the count here - showInactive: false, - }); - - if (agents.error) { - throw agents.error; - } - - const agentCount = agents.data?.total ?? 0; - - return { agentPolicyCount, agentCount }; -} diff --git a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts index 2adf814e8ffba8..b85b944c6ee00b 100644 --- a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts +++ b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts @@ -5,12 +5,11 @@ * 2.0. */ +import { LICENCE_FOR_MULTIPLE_AGENT_POLICIES } from '../../common/constants'; import { ExperimentalFeaturesService } from '../services'; import { useLicense } from './use_license'; -export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - export function useMultipleAgentPolicies() { const licenseService = useLicense(); const { enableReusableIntegrationPolicies } = ExperimentalFeaturesService.get(); diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index e2f8b883804841..80d8116baaaa33 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -65,6 +65,8 @@ export class PackagePolicyNameExistsError extends FleetError {} export class BundledPackageLocationNotFoundError extends FleetError {} export class PackagePolicyRequestError extends FleetError {} +export class PackagePolicyMultipleAgentPoliciesError extends FleetError {} +export class PackagePolicyOutputError extends FleetError {} export class EnrollmentKeyNameExistsError extends FleetError {} export class HostedAgentPolicyRestrictionRelatedError extends FleetError { diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 9e517ad928ba3c..9dfb920251e76e 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -182,6 +182,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked; @@ -202,6 +201,7 @@ describe('When calling package policy', () => { }; beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); // @ts-ignore const postMock = routerMock.versioned.post.mock; // @ts-ignore @@ -221,29 +221,6 @@ describe('When calling package policy', () => { }, }); }); - - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); }); describe('Update api handler', () => { @@ -308,6 +285,7 @@ describe('When calling package policy', () => { }); beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) => Promise.resolve(newData as PackagePolicy) ); @@ -393,58 +371,6 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.ok).toHaveBeenCalled(); - }); - - it('should throw if enterprise license and feature flag is disabled and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are not supported', - }, - }); - }); - - it('should throw if empty policy_ids are provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: [] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'At least one agent policy id must be provided', - }, - }); - }); - it('should rename the agentless agent policy to sync with the package policy name if agentless is enabled', async () => { jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any); jest diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 24872b2db72f20..31e38125dead5d 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -63,7 +63,6 @@ import { import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; import { - canUseMultipleAgentPolicies, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -242,21 +241,12 @@ export const createPackagePolicyHandler: FleetRequestHandler< const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username); let wasPackageAlreadyInstalled = false; - if ('output_id' in newPolicy) { - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - delete newPolicy.output_id; - } const spaceId = fleetContext.spaceId; try { if (!newPolicy.policy_id && (!newPolicy.policy_ids || newPolicy.policy_ids.length === 0)) { throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); } - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newPolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } - let newPackagePolicy: NewPackagePolicy; if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) { if (!pkg) { @@ -369,11 +359,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< try { const { force, package: pkg, ...body } = request.body; - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - if ('output_id' in body) { - delete body.output_id; - } - let newData: NewPackagePolicy; if ( @@ -420,10 +405,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } newData.inputs = alignInputsAndStreams(newData.inputs); - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } if (newData.policy_ids && newData.policy_ids.length === 0) { throw new PackagePolicyRequestError('At least one agent policy id must be provided'); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index f2ffc641c36ab5..032dab4a07acc7 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -21,8 +21,7 @@ import type { PackagePolicyInput, NewPackagePolicyInput, } from '../../../types'; -import { appContextService } from '../../../services'; -import { agentPolicyService, licenseService } from '../../../services'; +import { agentPolicyService } from '../../../services'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; import type { NewPackagePolicyInputStream } from '../../../../common'; @@ -56,20 +55,6 @@ export function removeFieldsFromInputSchema( }); } -const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - -export function canUseMultipleAgentPolicies() { - const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); - const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); - - return { - canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, - errorMessage: !hasEnterpriseLicence - ? 'Reusable integration policies are only available with an Enterprise license' - : 'Reusable integration policies are not supported', - }; -} - /** * If an agentless agent policy is associated with the package policy, * it will rename the agentless agent policy of a package policy to keep it in sync with the package policy name. diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 4b76a952a0f3b4..f6fcae4de6505c 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -481,6 +481,7 @@ export const getSavedObjectTypes = ( is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, @@ -639,6 +640,16 @@ export const getSavedObjectTypes = ( }, ], }, + '14': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + output_id: { type: 'keyword' }, + }, + }, + ], + }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts index cf835ec87c949a..d7e73d67a3c604 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts @@ -14,7 +14,6 @@ export const migratePackagePolicyToV850: SavedObjectMigrationFn { - // @ts-expect-error output_id property does not exists anymore delete packagePolicyDoc.attributes.output_id; return packagePolicyDoc; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 86f8126c1c45ae..8518e43fdaf0d2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -1,5 +1,173 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`getFullAgentPolicy should return the right outputs and permissions when package policies use their own outputs 1`] = ` +Object { + "agent": Object { + "download": Object { + "sourceURI": "http://default-registry.co", + }, + "features": Object {}, + "monitoring": Object { + "enabled": false, + "logs": false, + "metrics": false, + }, + "protection": Object { + "enabled": false, + "signing_key": "", + "uninstall_token_hash": "", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "integration-output-policy", + "inputs": Array [ + Object { + "data_stream": Object { + "namespace": "policyspace", + }, + "id": "test-logs-package-policy-using-output", + "meta": Object { + "package": Object { + "name": "test_package", + "version": "0.0.0", + }, + }, + "name": "test-policy-1", + "package_policy_id": "package-policy-using-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "test-remote-id", + }, + Object { + "data_stream": Object { + "namespace": "defaultspace", + }, + "id": "test-logs-package-policy-no-output", + "meta": Object { + "package": Object { + "name": "system", + "version": "1.0.0", + }, + }, + "name": "test-policy-2", + "package_policy_id": "package-policy-no-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "data-output-id", + }, + ], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-no-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-defaultspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_elastic_agent_monitoring": Object { + "indices": Array [ + Object { + "names": Array [], + "privileges": Array [], + }, + ], + }, + }, + "test-remote-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-using-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-policyspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "hosts": Array [ + "http://es-data.co:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "default": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "test-remote-id": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "service_token": undefined, + "type": "remote_elasticsearch", + }, + }, + "revision": 1, + "secret_references": Array [], + "signed": Object { + "data": "", + "signature": "", + }, +} +`; + exports[`getFullAgentPolicy should support a different data output 1`] = ` Object { "agent": Object { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 5701a60b56d037..6a084b5dde5865 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -401,6 +401,126 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy?.outputs['test-remote-id']).toBeDefined(); }); + it('should return the right outputs and permissions when package policies use their own outputs', async () => { + mockedGetPackageInfo.mockResolvedValue({ + data_streams: [ + { + type: 'logs', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'logs', + dataset: 'elastic_agent.filebeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.filebeat', + }, + ], + } as PackageInfo); + mockAgentPolicy({ + id: 'integration-output-policy', + status: 'active', + package_policies: [ + { + id: 'package-policy-using-output', + name: 'test-policy-1', + namespace: 'policyspace', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + output_id: 'test-remote-id', + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + { + id: 'package-policy-no-output', + name: 'test-policy-2', + namespace: '', + enabled: true, + package: { name: 'system', version: '1.0.0', title: 'System' }, + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ], + is_managed: false, + namespace: 'defaultspace', + revision: 1, + name: 'Policy', + updated_at: '2020-01-01', + updated_by: 'qwerty', + is_protected: false, + data_output_id: 'data-output-id', + }); + + const agentPolicy = await getFullAgentPolicy( + savedObjectsClientMock.create(), + 'integration-output-policy' + ); + expect(agentPolicy).toMatchSnapshot(); + }); + it('should return the sourceURI from the agent policy', async () => { mockAgentPolicy({ namespace: 'default', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index efc3a732149d64..0cfce9eb264f45 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -196,16 +196,46 @@ export async function getFullAgentPolicy( fullAgentPolicy.namespaces = [agentPolicy.space_id]; } - const dataPermissions = - (await storedPackagePoliciesToAgentPermissions( + const packagePoliciesByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = []; + return acc; + }, + {} + ); + (agentPolicy.package_policies || []).forEach((packagePolicy) => { + const packagePolicyDataOutput = packagePolicy.output_id + ? outputs.find((output) => output.id === packagePolicy.output_id) + : undefined; + if (packagePolicyDataOutput) { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(packagePolicyDataOutput)].push( + packagePolicy + ); + } else { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(dataOutput)].push(packagePolicy); + } + }); + + const dataPermissionsByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = {}; + return acc; + }, + {} + ); + for (const [outputId, packagePolicies] of Object.entries(packagePoliciesByOutputId)) { + const dataPermissions = await storedPackagePoliciesToAgentPermissions( packageInfoCache, agentPolicy.namespace, - agentPolicy.package_policies - )) || {}; - - dataPermissions._elastic_agent_checks = { - cluster: DEFAULT_CLUSTER_PERMISSIONS, - }; + packagePolicies + ); + dataPermissionsByOutputId[outputId] = { + _elastic_agent_checks: { + cluster: DEFAULT_CLUSTER_PERMISSIONS, + }, + ...(dataPermissions || {}), + }; + } const monitoringPermissions = await getMonitoringPermissions( soClient, @@ -233,8 +263,11 @@ export async function getFullAgentPolicy( Object.assign(permissions, monitoringPermissions); } - if (outputId === getOutputIdForAgentPolicy(dataOutput)) { - Object.assign(permissions, dataPermissions); + if ( + outputId === getOutputIdForAgentPolicy(dataOutput) || + packagePoliciesByOutputId[outputId].length > 0 + ) { + Object.assign(permissions, dataPermissionsByOutputId[outputId]); } outputPermissions[outputId] = permissions; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index dfdcf26adaa4ed..0afef0170a99f1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -13,7 +13,7 @@ import { appContextService } from '..'; import { outputService } from '../output'; import { validateOutputForPolicy } from '.'; -import { validateOutputForNewPackagePolicy } from './outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './outputs_helpers'; jest.mock('../app_context'); jest.mock('../output'); @@ -254,14 +254,14 @@ describe('validateOutputForPolicy', () => { }); }); -describe('validateOutputForNewPackagePolicy', () => { - it('should not allow fleet_server integration to be added to a policy using a logstash output', async () => { +describe('validateAgentPolicyOutputForIntegration', () => { + it('should not allow fleet_server integration to be added or edited to a policy using a logstash output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'logstash', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -273,15 +273,29 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "fleet_server" cannot be added to agent policy "Agent policy" because it uses output type "logstash".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'fleet_server', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "logstash" which cannot be used for integration "fleet_server".' + ); }); - it('should not allow apm integration to be added to a policy using a kafka output', async () => { + it('should not allow apm integration to be added or edited to a policy using a kafka output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'kafka', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -293,6 +307,20 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "apm" cannot be added to agent policy "Agent policy" because it uses output type "kafka".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'apm', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "kafka" which cannot be used for integration "apm".' + ); }); it('should not allow synthetics integration to be added to a policy using a default logstash output', async () => { @@ -302,7 +330,7 @@ describe('validateOutputForNewPackagePolicy', () => { } as any); mockedOutputService.getDefaultDataOutputId.mockResolvedValue('default'); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -320,7 +348,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'logstash', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -335,7 +363,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'elasticsearch', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index 67f5a7772aa52a..66f45427601e6e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -96,10 +96,11 @@ export async function validateOutputForPolicy( } } -export async function validateOutputForNewPackagePolicy( +export async function validateAgentPolicyOutputForIntegration( soClient: SavedObjectsClientContract, agentPolicy: AgentPolicy, - packageName: string + packageName: string, + isNewPackagePolicy: boolean = true ) { const allowedOutputTypeForPolicy = getAllowedOutputTypesForIntegration(packageName); @@ -109,9 +110,15 @@ export async function validateOutputForNewPackagePolicy( if (isOutputTypeRestricted) { const dataOutput = await getDataOutputForAgentPolicy(soClient, agentPolicy); if (!allowedOutputTypeForPolicy.includes(dataOutput.type)) { - throw new OutputInvalidError( - `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` - ); + if (isNewPackagePolicy) { + throw new OutputInvalidError( + `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` + ); + } else { + throw new OutputInvalidError( + `Agent policy "${agentPolicy.name}" uses output type "${dataOutput.type}" which cannot be used for integration "${packageName}".` + ); + } } } } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts index 40cda7583a3c15..9004d544f399db 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts @@ -222,6 +222,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { version: '0.0.0', }, inputs: [mockInput, mockInput2], + output_id: 'new-output', }, { ...mockPackagePolicy, @@ -243,7 +244,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-logs', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', @@ -270,7 +271,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-metrics', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index d7f0c70a0786b7..807312fe5e7cb2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -27,7 +27,7 @@ const isPolicyEnabled = (packagePolicy: PackagePolicy) => { export const storedPackagePolicyToAgentInputs = ( packagePolicy: PackagePolicy, packageInfo?: PackageInfo, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, addFields?: FullAgentPolicyAddFields ): FullAgentPolicyInput[] => { @@ -62,7 +62,7 @@ export const storedPackagePolicyToAgentInputs = ( data_stream: { namespace: packagePolicy?.namespace || agentPolicyNamespace || 'default', // custom namespace has precedence on agent policy's one }, - use_output: outputId, + use_output: packagePolicy.output_id || agentPolicyOutputId, package_policy_id: packagePolicy.id, ...getFullInputStreams(input), }; @@ -140,7 +140,7 @@ export const getFullInputStreams = ( export const storedPackagePoliciesToAgentInputs = async ( packagePolicies: PackagePolicy[], packageInfoCache: Map, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, globalDataTags?: GlobalDataTag[] ): Promise => { @@ -164,7 +164,7 @@ export const storedPackagePoliciesToAgentInputs = async ( ...storedPackagePolicyToAgentInputs( packagePolicy, packageInfo, - outputId, + agentPolicyOutputId, agentPolicyNamespace, addFields ) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts index 0108e9cd97721c..52dd34a757693b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts @@ -35,11 +35,20 @@ export async function fetchRelatedSavedObjects( const monitoringOutputId = agentPolicy.monitoring_output_id || defaultMonitoringOutputId || dataOutputId; + const outputIds = uniq([ + dataOutputId, + monitoringOutputId, + ...(agentPolicy.package_policies || []).reduce((acc: string[], packagePolicy) => { + if (packagePolicy.output_id) { + acc.push(packagePolicy.output_id); + } + return acc; + }, []), + ]); + const [outputs, { host: downloadSourceUri, proxy_id: downloadSourceProxyId }, fleetServerHosts] = await Promise.all([ - outputService.bulkGet(soClient, uniq([dataOutputId, monitoringOutputId]), { - ignoreNotFound: true, - }), + outputService.bulkGet(soClient, outputIds, { ignoreNotFound: true }), getSourceUriForAgentPolicy(soClient, agentPolicy), getFleetServerHostsForAgentPolicy(soClient, agentPolicy).catch((err) => { appContextService diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index a5dbbc6b233b3b..2c2c747681887f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -23,7 +23,7 @@ import type { NewAgentPolicy, PreconfiguredAgentPolicy, } from '../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { AGENT_POLICY_INDEX, SO_SEARCH_LIMIT } from '../../common'; @@ -43,32 +43,63 @@ import type { UninstallTokenServiceInterface } from './security/uninstall_token_ function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); + const mockPolicy = { + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes: agentPolicyAttributes as AgentPolicy, + }; mock.get.mockImplementation(async (type: string, id: string) => { return { - type, id, - references: [], - attributes: agentPolicyAttributes as AgentPolicy, + ...mockPolicy, }; }); - mock.find.mockImplementation(async (options) => { + mock.bulkGet.mockImplementation(async (options) => { return { - saved_objects: [ - { - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - attributes: { - fleet_server_hosts: ['http://fleetserver:8220'], - }, - type: 'ingest_manager_settings', - score: 1, - references: [], - }, - ], - total: 1, - page: 1, - per_page: 1, + saved_objects: [], }; }); + mock.find.mockImplementation(async (options) => { + switch (options.type) { + case AGENT_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [ + { + id: 'agent-policy-id', + score: 1, + ...mockPolicy, + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + case PACKAGE_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [], + total: 0, + page: 1, + per_page: 1, + }; + default: + return { + saved_objects: [ + { + id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', + attributes: { + fleet_server_hosts: ['http://fleetserver:8220'], + }, + type: 'ingest_manager_settings', + score: 1, + references: [], + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + } + }); return mock; } @@ -754,12 +785,12 @@ describe('Agent policy', () => { [ expect.objectContaining({ attributes: expect.objectContaining({ - fleet_server_hosts: ['http://fleetserver:8220'], - revision: NaN, - updated_by: 'system', + monitoring_enabled: ['metrics'], }), - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - type: 'ingest_manager_settings', + id: 'agent-policy-id', + namespace: undefined, + type: 'ingest-agent-policies', + version: undefined, }), ], expect.objectContaining({ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index d243ef8b60e168..42bede95986b6c 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -16,7 +16,7 @@ import type { SavedObjectsBulkUpdateObject, SavedObjectsBulkUpdateResponse, SavedObjectsClientContract, - SavedObjectsFindResult, + SavedObject, SavedObjectsUpdateResponse, } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; @@ -41,6 +41,7 @@ import { import type { HTTPAuthorizationHeader } from '../../common/http_authorization_header'; import { + PACKAGE_POLICY_SAVED_OBJECT_TYPE, AGENT_POLICY_SAVED_OBJECT_TYPE, AGENTS_PREFIX, FLEET_AGENT_POLICIES_SCHEMA_VERSION, @@ -56,6 +57,7 @@ import type { NewAgentPolicy, NewPackagePolicy, PackagePolicy, + PackagePolicySOAttributes, PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, @@ -876,7 +878,7 @@ class AgentPolicyService { private async _bumpPolicies( internalSoClientWithoutSpaceExtension: SavedObjectsClientContract, esClient: ElasticsearchClient, - savedObjectsResults: Array>, + savedObjectsResults: Array>, options?: { user?: AuthenticatedUser } ): Promise> { const bumpedPolicies = savedObjectsResults.map( @@ -938,10 +940,12 @@ class AgentPolicyService { outputId: string, options?: { user?: AuthenticatedUser } ): Promise> { + const { useSpaceAwareness } = appContextService.getExperimentalFeatures(); const internalSoClientWithoutSpaceExtension = appContextService.getInternalUserSOClientWithoutSpaceExtension(); - const currentPolicies = + // All agent policies directly using output + const agentPoliciesUsingOutput = await internalSoClientWithoutSpaceExtension.find({ type: SAVED_OBJECT_TYPE, fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], @@ -950,10 +954,47 @@ class AgentPolicyService { perPage: SO_SEARCH_LIMIT, namespaces: ['*'], }); + + // All package policies directly using output + const packagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.find({ + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + fields: ['output_id', 'namespaces', 'policy_ids'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + namespaces: ['*'], + }); + + const agentPolicyIdsDirectlyUsingOutput = agentPoliciesUsingOutput.saved_objects.map( + (agentPolicySO) => agentPolicySO.id + ); + const agentPolicyIdsOfPackagePoliciesUsingOutput = + packagePoliciesUsingOutput.saved_objects.reduce((acc: Set, packagePolicySO) => { + const newIds = packagePolicySO.attributes.policy_ids.filter((policyId) => { + return !agentPolicyIdsDirectlyUsingOutput.includes(policyId); + }); + return new Set([...acc, ...newIds]); + }, new Set()); + + // Agent policies of the identified package policies, excluding ones already retrieved directly + const agentPoliciesOfPackagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.bulkGet( + [...agentPolicyIdsOfPackagePoliciesUsingOutput].map((id) => ({ + type: SAVED_OBJECT_TYPE, + id, + fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], + ...(useSpaceAwareness ? { namespaces: ['*'] } : {}), + })) + ); + return this._bumpPolicies( internalSoClientWithoutSpaceExtension, esClient, - currentPolicies.saved_objects, + [ + ...agentPoliciesUsingOutput.saved_objects, + ...agentPoliciesOfPackagePoliciesUsingOutput.saved_objects, + ], options ); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 3bc9003162f448..0951657fae6dd0 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -19,10 +19,12 @@ import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { auditLoggingService } from './audit_logging'; jest.mock('./app_context'); jest.mock('./agent_policy'); +jest.mock('./package_policy'); jest.mock('./audit_logging'); const mockedAuditLoggingService = auditLoggingService as jest.Mocked; @@ -43,6 +45,7 @@ mockedAppContextService.getLogger.mockImplementation(() => { mockedAppContextService.getExperimentalFeatures.mockReturnValue({} as any); const mockedAgentPolicyService = agentPolicyService as jest.Mocked; +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; const CLOUD_ID = 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw=='; @@ -219,6 +222,7 @@ function getMockedSoClient( }); mockedAppContextService.getInternalUserSOClient.mockReturnValue(soClient); + mockedAppContextService.getInternalUserSOClientWithoutSpaceExtension.mockReturnValue(soClient); return soClient; } @@ -288,17 +292,23 @@ describe('Output Service', () => { } as unknown as ReturnType; beforeEach(() => { + mockedAgentPolicyService.getByIDs.mockResolvedValue([]); mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.hasFleetServerIntegration.mockClear(); mockedAgentPolicyService.hasSyntheticsIntegration.mockClear(); mockedAgentPolicyService.removeOutputFromAll.mockReset(); + mockedPackagePolicyService.removeOutputFromAll.mockReset(); mockedAppContextService.getInternalUserSOClient.mockReset(); mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); mockedAgentPolicyService.update.mockReset(); }); + afterEach(() => { + mockedAgentPolicyService.getByIDs.mockClear(); + }); + describe('create', () => { describe('elasticsearch output', () => { it('works with a predefined id', async () => { @@ -1412,7 +1422,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new Elasticsearch output.' ); }); @@ -1428,7 +1438,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new Elasticsearch output.' ); }); @@ -1758,6 +1768,7 @@ describe('Output Service', () => { fromPreconfiguration: true, }); expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled(); + expect(mockedPackagePolicyService.removeOutputFromAll).toBeCalled(); expect(soClient.delete).toBeCalled(); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f42867f0686bb9..2748ad78e765b9 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -38,6 +38,7 @@ import type { } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE, @@ -63,6 +64,7 @@ import { import type { OutputType } from '../types'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; import { auditLoggingService } from './audit_logging'; @@ -124,40 +126,63 @@ function outputSavedObjectToOutput(so: SavedObject): Output }; } -async function getAgentPoliciesPerOutput( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - let kuery: string; +async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + let agentPoliciesKuery: string; + const packagePoliciesKuery: string = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${outputId}"`; if (outputId) { if (isDefault) { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; } } else { if (isDefault) { - kuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { return; } } - const agentPolicySO = await agentPolicyService.list(soClient, { - kuery, + // Get agent policies directly using output + const directAgentPolicies = await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: agentPoliciesKuery, perPage: SO_SEARCH_LIMIT, withPackagePolicies: true, }); - return agentPolicySO?.items; + const directAgentPolicyIds = directAgentPolicies?.items.map((policy) => policy.id); + + // Get package policies using output and derive agent policies from that which + // are not already identfied above. The IDs cannot be used as part of the kuery + // above since the underlying saved object client .find() only filters on attributes + const packagePolicySOs = await packagePolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: packagePoliciesKuery, + perPage: SO_SEARCH_LIMIT, + }); + const agentPolicyIdsFromPackagePolicies = [ + ...new Set( + packagePolicySOs?.items.reduce((acc: string[], packagePolicy) => { + return [ + ...acc, + ...packagePolicy.policy_ids.filter((id) => !directAgentPolicyIds?.includes(id)), + ]; + }, []) + ), + ]; + const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( + internalSoClientWithoutSpaceExtension, + agentPolicyIdsFromPackagePolicies, + { + withPackagePolicies: true, + } + ); + + return [...directAgentPolicies.items, ...agentPoliciesFromPackagePolicies]; } -async function validateLogstashOutputNotUsedInAPMPolicy( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - const agentPolicies = await getAgentPoliciesPerOutput(soClient, outputId, isDefault); +async function validateLogstashOutputNotUsedInAPMPolicy(outputId?: string, isDefault?: boolean) { + const agentPolicies = await getAgentPoliciesPerOutput(outputId, isDefault); // Validate no policy with APM use that policy if (agentPolicies) { @@ -169,16 +194,19 @@ async function validateLogstashOutputNotUsedInAPMPolicy( } } -async function findPoliciesWithFleetServerOrSynthetics( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { +async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + // find agent policies by outputId // otherwise query all the policies const agentPolicies = outputId - ? await getAgentPoliciesPerOutput(soClient, outputId, isDefault) - : (await agentPolicyService.list(soClient, { withPackagePolicies: true }))?.items; + ? await getAgentPoliciesPerOutput(outputId, isDefault) + : ( + await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + withPackagePolicies: true, + }) + )?.items; const policiesWithFleetServer = agentPolicies?.filter((policy) => agentPolicyService.hasFleetServerIntegration(policy)) || []; @@ -199,13 +227,12 @@ function validateOutputNotUsedInPolicy( dataOutputType )} output cannot be used with ${integrationName} integration in ${ agentPolicy.name - }. Please create a new ElasticSearch output.` + }. Please create a new Elasticsearch output.` ); } } async function validateTypeChanges( - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, id: string, data: Nullable>, @@ -213,12 +240,14 @@ async function validateTypeChanges( defaultDataOutputId: string | null, fromPreconfiguration: boolean ) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); const mergedIsDefault = data.is_default ?? originalOutput.is_default; const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient, id, mergedIsDefault); + await findPoliciesWithFleetServerOrSynthetics(id, mergedIsDefault); if (data.type === outputType.Logstash || originalOutput.type === outputType.Logstash) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, id, mergedIsDefault); + await validateLogstashOutputNotUsedInAPMPolicy(id, mergedIsDefault); } // prevent changing an ES output to logstash or kafka if it's used by fleet server or synthetics policies if ( @@ -230,7 +259,7 @@ async function validateTypeChanges( validateOutputNotUsedInPolicy(policiesWithSynthetics, data.type, 'Synthetics'); } await updateAgentPoliciesDataOutputId( - soClient, + internalSoClientWithoutSpaceExtension, esClient, data, mergedIsDefault, @@ -274,7 +303,7 @@ class OutputService { return appContextService.getInternalUserSOClient(fakeRequest); } - private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { + private async _getDefaultDataOutputsSO() { const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -403,7 +432,7 @@ class OutputService { } public async getDefaultDataOutputId(soClient: SavedObjectsClientContract) { - const outputs = await this._getDefaultDataOutputsSO(soClient); + const outputs = await this._getDefaultDataOutputsSO(); if (!outputs.saved_objects.length) { return null; @@ -454,7 +483,7 @@ class OutputService { const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, undefined, data.is_default); + await validateLogstashOutputNotUsedInAPMPolicy(undefined, data.is_default); if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { throw new FleetEncryptedSavedObjectEncryptionKeyRequired( `${output.type} output needs encrypted saved object api key to be set` @@ -462,7 +491,7 @@ class OutputService { } } const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient); + await findPoliciesWithFleetServerOrSynthetics(); await updateAgentPoliciesDataOutputId( soClient, esClient, @@ -740,8 +769,17 @@ class OutputService { throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`); } + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + + await packagePolicyService.removeOutputFromAll( + internalSoClientWithoutSpaceExtension, + appContextService.getInternalUserESClient(), + id + ); + await agentPolicyService.removeOutputFromAll( - soClient, + internalSoClientWithoutSpaceExtension, appContextService.getInternalUserESClient(), id ); @@ -808,7 +846,6 @@ class OutputService { const mergedType = data.type ?? originalOutput.type; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); await validateTypeChanges( - soClient, esClient, id, updateData, diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts index 9bf58298b09d35..311bfeb339f20f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts @@ -4,10 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import { PackagePolicyMocks } from '../../mocks'; -import { mapPackagePolicySavedObjectToPackagePolicy } from './utils'; +import { appContextService } from '../app_context'; +import { licenseService } from '../license'; +import { outputService } from '../output'; + +import { mapPackagePolicySavedObjectToPackagePolicy, preflightCheckPackagePolicy } from './utils'; describe('Package Policy Utils', () => { describe('mapPackagePolicySavedObjectToPackagePolicy()', () => { @@ -38,6 +43,7 @@ describe('Package Policy Utils', () => { }, policy_id: '444-555-666', policy_ids: ['444-555-666'], + output_id: 'output-123', revision: 1, secret_references: [], updated_at: '2024-01-25T15:21:13.389Z', @@ -47,4 +53,94 @@ describe('Package Policy Utils', () => { }); }); }); + + describe('preflightCheckPackagePolicy', () => { + beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); + jest.spyOn(appContextService, 'getExperimentalFeatures').mockClear(); + }); + const soClient = savedObjectsClientMock.create(); + const testPolicy = { + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_ids: ['test'], + inputs: [], + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + }; + + it('should throw if no enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError( + 'Reusable integration policies are only available with an Enterprise license' + ); + }); + + it('should throw if enterprise license and multiple policy_ids is provided but no feature flag', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError('Reusable integration policies are not supported'); + }); + + it('should not throw if enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).resolves.not.toThrow(); + }); + + it('should throw if no valid license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, output_id: 'some-output' }) + ).rejects.toThrowError('Output per integration is only available with an enterprise license'); + }); + + it('should throw if valid license and an incompatible output_id for the package is given', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'non-es-output', + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + }) + ).rejects.toThrowError('Output type "kafka" is not usable with package "apm"'); + }); + + it('should not throw if valid license and valid output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'es-output', type: 'elasticsearch' } as any); + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'es-output', + }) + ).resolves.not.toThrow(); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index cf2d69c8874b96..61ff380fec559f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -6,8 +6,18 @@ */ import type { SavedObject } from '@kbn/core-saved-objects-common/src/server_types'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { PackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { + LICENCE_FOR_OUTPUT_PER_INTEGRATION, + LICENCE_FOR_MULTIPLE_AGENT_POLICIES, +} from '../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../common/services/output_helpers'; +import type { PackagePolicy, NewPackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { PackagePolicyMultipleAgentPoliciesError, PackagePolicyOutputError } from '../../errors'; +import { licenseService } from '../license'; +import { outputService } from '../output'; +import { appContextService } from '../app_context'; export const mapPackagePolicySavedObjectToPackagePolicy = ({ /* eslint-disable @typescript-eslint/naming-convention */ @@ -21,6 +31,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, // `package` is a reserved keyword package: packageInfo, inputs, @@ -45,6 +56,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, package: packageInfo, inputs, vars, @@ -59,3 +71,69 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ created_by, }; }; + +export async function preflightCheckPackagePolicy( + soClient: SavedObjectsClientContract, + packagePolicy: PackagePolicy | NewPackagePolicy +) { + // If package policy has multiple agent policies, check if they can be used + const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = + canUseMultipleAgentPolicies(); + if ((packagePolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyMultipleAgentPoliciesError(canUseMultipleAgentPoliciesErrorMessage); + } + + // If package policy has an output_id, check if it can be used + if (packagePolicy.output_id && packagePolicy.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration( + soClient, + packagePolicy.package.name, + packagePolicy.output_id + ); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyOutputError(outputForIntegrationErrorMessage); + } + } +} + +export function canUseMultipleAgentPolicies() { + const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); + const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); + + return { + canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, + errorMessage: !hasEnterpriseLicence + ? 'Reusable integration policies are only available with an Enterprise license' + : 'Reusable integration policies are not supported', + }; +} + +export async function canUseOutputForIntegration( + soClient: SavedObjectsClientContract, + packageName: string, + outputId: string +) { + const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasAllowedLicense) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, + }; + } + + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const output = await outputService.get(soClient, outputId); + + if (!allowedOutputTypes.includes(output.type)) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, + }; + } + + return { + canUseOutputForIntegrationResult: true, + errorMessage: null, + }; +} diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index a62a5048ec567d..9505edd3556cb3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -2921,6 +2921,7 @@ describe('Package policy service', () => { version: 'WzYyMzcsMV0=', name: 'my-cis_kubernetes_benchmark', namespace: 'default', + output_id: null, description: '', package: { name: 'cis_kubernetes_benchmark', @@ -5096,6 +5097,61 @@ describe('Package policy service', () => { ); }); }); + + describe('removeOutputFromAll', () => { + it('should update policies using deleted output', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const updateSpy = jest.spyOn(packagePolicyService, 'update'); + soClient.find.mockResolvedValue({ + saved_objects: [ + { + id: 'package-policy-1', + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }, + ], + } as any); + soClient.get.mockImplementation((type, id, options): any => { + if (id === 'package-policy-1') { + return Promise.resolve({ + id, + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }); + } + }); + + await packagePolicyService.removeOutputFromAll(soClient, esClient, 'output-id-123'); + + expect(updateSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'package-policy-1', + { + name: 'policy1', + enabled: true, + policy_id: 'agent-policy-1', + policy_ids: ['agent-policy-1'], + output_id: null, + inputs: [], + } + ); + }); + }); }); describe('getUpgradeDryRunDiff', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index d9605880c1572e..65495d51f0f60c 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -115,6 +115,7 @@ import { sendTelemetryEvents } from './upgrade_sender'; import { handleExperimentalDatastreamFeatureOptIn, mapPackagePolicySavedObjectToPackagePolicy, + preflightCheckPackagePolicy, } from './package_policies'; import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { @@ -131,7 +132,7 @@ import { isSecretStorageEnabled, } from './secrets'; import { getPackageAssetsMap } from './epm/packages/get'; -import { validateOutputForNewPackagePolicy } from './agent_policies/outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './agent_policies/outputs_helpers'; import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; @@ -212,6 +213,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.debug(`Creating new package policy`); this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); let enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', @@ -228,8 +230,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const agentPolicy = await agentPolicyService.get(soClient, policyId, true); agentPolicies.push(agentPolicy); - if (agentPolicy && enrichedPackagePolicy.package?.name) { - await validateOutputForNewPackagePolicy( + // If package policy did not set an output_id, see if the agent policy's output is compatible + if (!packagePolicy.output_id && agentPolicy && enrichedPackagePolicy.package?.name) { + await validateAgentPolicyOutputForIntegration( soClient, agentPolicy, enrichedPackagePolicy.package?.name @@ -408,14 +411,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!packagePolicy.id) { packagePolicy.id = SavedObjectsUtils.generateId(); } - - this.keepPolicyIdInSync(packagePolicy); - auditLoggingService.writeCustomSoAuditLog({ action: 'create', id: packagePolicy.id, savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, }); + + this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); } const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids)); @@ -471,6 +474,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const { pkgInfo, assetsMap } = packageInfoAndAsset; + validatePackagePolicyOrThrow(packagePolicy, pkgInfo); inputs = pkgInfo ? await _compilePackagePolicyInputs( @@ -822,6 +826,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { try { logger.debug(`Starting update of package policy ${id}`); + await preflightCheckPackagePolicy(soClient, packagePolicyUpdate); enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyUpdate', packagePolicyUpdate, @@ -1052,6 +1057,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const id = packagePolicyUpdate.id; this.keepPolicyIdInSync(packagePolicyUpdate); const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; + await preflightCheckPackagePolicy(soClient, packagePolicy); const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); if (!oldPackagePolicy) { throw new PackagePolicyNotFoundError('Package policy not found'); @@ -1762,6 +1768,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }, policy_id: newPolicy.policy_id ?? agentPolicyId, policy_ids: newPolicy.policy_ids ?? [agentPolicyId], + output_id: newPolicy.output_id, inputs: newPolicy.inputs[0]?.streams ? newPolicy.inputs : inputs, vars: newPolicy.vars || newPP.vars, }; @@ -1982,6 +1989,76 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } + public async removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ) { + const packagePolicies = ( + await soClient.find({ + type: SAVED_OBJECT_TYPE, + fields: ['name', 'enabled', 'policy_ids', 'inputs', 'output_id'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + }) + ).saved_objects.map(mapPackagePolicySavedObjectToPackagePolicy); + + if (packagePolicies.length > 0) { + const getPackagePolicyUpdate = (packagePolicy: PackagePolicy) => ({ + name: packagePolicy.name, + enabled: packagePolicy.enabled, + policy_ids: packagePolicy.policy_ids, + inputs: packagePolicy.inputs, + output_id: packagePolicy.output_id === outputId ? null : packagePolicy.output_id, + }); + + // Validate that the new cleared/default output is valid for the package policies + // (from each of the associated agent policies) before updating any of them + await pMap( + packagePolicies, + async (packagePolicy) => { + const existingPackagePolicy = await this.get(soClient, packagePolicy.id); + + if (!existingPackagePolicy) { + throw new PackagePolicyNotFoundError('Package policy not found'); + } + + for (const policyId of packagePolicy.policy_ids) { + if (packagePolicy.package?.name) { + const agentPolicy = await agentPolicyService.get(soClient, policyId, true); + if (agentPolicy) { + await validateAgentPolicyOutputForIntegration( + soClient, + agentPolicy, + packagePolicy.package.name, + false + ); + } + } + } + }, + { + concurrency: 50, + } + ); + await pMap( + packagePolicies, + (packagePolicy) => { + return this.update( + soClient, + esClient, + packagePolicy.id, + getPackagePolicyUpdate(packagePolicy) + ); + }, + { + concurrency: 50, + } + ); + } + } + fetchAllItemIds( soClient: SavedObjectsClientContract, { perPage = 1000, kuery }: PackagePolicyClientFetchAllItemIdsOptions = {} diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 1462d451388b52..fed46872ab6cba 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -219,6 +219,18 @@ export interface PackagePolicyClient { experimentalDataStreamFeatures: ExperimentalDataStreamFeature[]; }>; + /** + * Remove an output from all package policies that are using it, and replace the output by the default ones. + * @param soClient + * @param esClient + * @param outputId + */ + removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ): Promise; + /** * Returns an `AsyncIterable` for retrieving all integration policy IDs * @param soClient diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index f6feffd24df535..3088814c8f8a32 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -6,6 +6,7 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + import { appContextService } from '..'; import type { PreconfiguredOutput } from '../../../common/types'; @@ -31,6 +32,9 @@ const mockedOutputService = outputService as jest.Mocked; jest.mock('../app_context', () => ({ appContextService: { getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + getExperimentalFeatures: () => ({ + useSpaceAwareness: true, + }), getLogger: () => new Proxy( {}, @@ -60,6 +64,9 @@ describe('output preconfiguration', () => { per_page: 0, total: 0, }); + internalSoClientWithoutSpaceExtension.bulkGet.mockResolvedValue({ + saved_objects: [], + }); mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.delete.mockReset(); diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 54db36b49d0430..c80cb45c84ff9a 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -98,6 +98,7 @@ const PackagePolicyBaseSchema = { namespace: schema.maybe(PackagePolicyNamespaceSchema), policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + output_id: schema.nullable(schema.maybe(schema.string())), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), package: schema.maybe( @@ -109,8 +110,6 @@ const PackagePolicyBaseSchema = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)), vars: schema.maybe(ConfigRecordSchema), overrides: schema.maybe( @@ -152,8 +151,6 @@ const CreatePackagePolicyProps = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -191,6 +188,7 @@ export const SimplifiedPackagePolicyBaseSchema = schema.object({ name: schema.string(), description: schema.maybe(schema.string()), namespace: schema.maybe(schema.string()), + output_id: schema.nullable(schema.maybe(schema.string())), vars: schema.maybe(SimplifiedVarsSchema), inputs: schema.maybe( schema.recordOf( diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 6216eb4bbd3269..79586b1885ed97 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -162,6 +162,7 @@ export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( }), description: schema.maybe(schema.string()), namespace: schema.maybe(PackagePolicyNamespaceSchema), + output_id: schema.nullable(schema.maybe(schema.string())), inputs: schema.maybe( schema.arrayOf( schema.object({ diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 79640a0f90e129..ef39759b206f55 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -121,6 +121,8 @@ export interface PackagePolicySOAttributes { inputs: PackagePolicyInput[]; policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; updated_at: string; updated_by: string; description?: string; diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap index 9112ad20ad8609..a4d255613133ef 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap @@ -30,6 +30,7 @@ Object { "enabled": true, "name": "system-1", "namespace": "default", + "output_id": null, "package": Object { "name": "system", "requires_root": true, diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 76585c1f0099ad..b860a774ba1220 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -120,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { } }; - const createAgentPolicy = async (spaceId?: string): Promise => { + const createAgentPolicy = async ( + spaceId?: string, + dataOutputId?: string + ): Promise => { const { body: testPolicyRes } = await supertest .post(spaceId ? `/s/${spaceId}/api/fleet/agent_policies` : `/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') @@ -128,6 +131,7 @@ export default function (providerContext: FtrProviderContext) { name: `test ${uuidV4()}`, description: '', namespace: 'default', + ...(dataOutputId ? { data_output_id: dataOutputId } : {}), }) .expect(200); @@ -148,6 +152,33 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }; + const createPackagePolicy = async ( + agentPolicyIds: string[], + spaceId?: string, + outputId?: string + ): Promise => { + const { body: testPolicyRes } = await supertest + .post(spaceId ? `/s/${spaceId}/api/fleet/package_policies` : `/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `package ${uuidV4()}`, + description: '', + namespace: 'default', + policy_ids: agentPolicyIds, + enabled: true, + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + ...(outputId ? { output_id: outputId } : {}), + }) + .expect(200); + + return testPolicyRes; + }; + const getAgentPolicy = async ( policyId: string, spaceId?: string @@ -396,7 +427,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -417,7 +448,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -765,10 +796,28 @@ export default function (providerContext: FtrProviderContext) { }); it('should bump all policies in all spaces if updating the default output', async () => { - const [policy1, policy2, policy3] = await Promise.all([ - createAgentPolicy(), + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies with default output under agent policies not using default output + // to ensure that those agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy2.item.id], undefined, defaultOutputId), + createPackagePolicy([policy4.item.id], TEST_SPACE_ID, defaultOutputId), ]); await supertest @@ -781,20 +830,79 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); - const [updatedPolicy1, updatedPolicy2, updatedPolicy3] = await Promise.all([ + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ getAgentPolicy(policy1.item.id), getAgentPolicy(policy2.item.id), getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 1); - expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 2); expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 1); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 2); + + // cleanup + await Promise.all([ + deleteAgentPolicy(policy1.item.id), + deleteAgentPolicy(policy2.item.id), + deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + }); + + it('should bump all policies in all spaces if updating non-default output', async () => { + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ + createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), + createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies under agent policies using default output to ensure those + // agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy1.item.id], undefined, nonDefaultOutput.item.id), + createPackagePolicy([policy3.item.id], TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + await supertest + .put(`/api/fleet/outputs/${nonDefaultOutput.item.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Nondefault Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + }) + .expect(200); + + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ + getAgentPolicy(policy1.item.id), + getAgentPolicy(policy2.item.id), + getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + + expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 2); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 2); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 1); // cleanup await Promise.all([ deleteAgentPolicy(policy1.item.id), deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); }); });