diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json index dd233e14648bd7..211777a5274a5b 100644 --- a/packages/kbn-check-mappings-update-cli/current_fields.json +++ b/packages/kbn-check-mappings-update-cli/current_fields.json @@ -630,6 +630,7 @@ "is_managed", "name", "namespace", + "output_id", "overrides", "package", "package.name", diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json index 4ded89844052be..e6e1fef63ee85a 100644 --- a/packages/kbn-check-mappings-update-cli/current_mappings.json +++ b/packages/kbn-check-mappings-update-cli/current_mappings.json @@ -2101,6 +2101,9 @@ "namespace": { "type": "keyword" }, + "output_id": { + "type": "keyword" + }, "overrides": { "index": false, "type": "flattened" diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts index af58fd9d383bc6..6d978b2d33ca45 100644 --- a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts +++ b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts @@ -119,7 +119,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "90625b4a5ded9d4867358fcccc14a57c0454fcee", "ingest-download-sources": "279a68147e62e4d8858c09ad1cf03bd5551ce58d", "ingest-outputs": "daafff49255ab700e07491376fe89f04fc998b91", - "ingest-package-policies": "2c0f7c72d211bb7d3076ce2fc0bd368f9c16d274", + "ingest-package-policies": "53a94064674835fdb35e5186233bcd7052eabd22", "ingest_manager_settings": "91445219e7115ff0c45d1dabd5d614a80b421797", "inventory-view": "b8683c8e352a286b4aca1ab21003115a4800af83", "kql-telemetry": "93c1d16c1a0dfca9c8842062cf5ef8f62ae401ad", diff --git a/x-pack/plugins/fleet/common/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts index f804dc872e0d18..6499da7f86cc9a 100644 --- a/x-pack/plugins/fleet/common/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -6,7 +6,7 @@ */ /** - * ATTENTION: Mappings for Fleet are defined in the ElasticSearch repo. + * ATTENTION: Mappings for Fleet are defined in the Elasticsearch repo. * * The following mappings declared here closely mirror them * But they are only used to perform validation on the endpoints using ListWithKuery @@ -54,6 +54,7 @@ export const PACKAGE_POLICIES_MAPPINGS = { is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, diff --git a/x-pack/plugins/fleet/common/constants/output.ts b/x-pack/plugins/fleet/common/constants/output.ts index 8ddb8ed4d2f064..fb01ba991d3d26 100644 --- a/x-pack/plugins/fleet/common/constants/output.ts +++ b/x-pack/plugins/fleet/common/constants/output.ts @@ -29,6 +29,7 @@ export const DEFAULT_OUTPUT: NewOutput = { export const SERVERLESS_DEFAULT_OUTPUT_ID = 'es-default-output'; export const LICENCE_FOR_PER_POLICY_OUTPUT = 'platinum'; +export const LICENCE_FOR_OUTPUT_PER_INTEGRATION = 'enterprise'; /** * Kafka constants diff --git a/x-pack/plugins/fleet/common/constants/package_policy.ts b/x-pack/plugins/fleet/common/constants/package_policy.ts index 1645d1a86efb72..00b41a8a29de2f 100644 --- a/x-pack/plugins/fleet/common/constants/package_policy.ts +++ b/x-pack/plugins/fleet/common/constants/package_policy.ts @@ -13,3 +13,5 @@ export const inputsFormat = { Simplified: 'simplified', Legacy: 'legacy', } as const; + +export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; diff --git a/x-pack/plugins/fleet/common/openapi/bundled.json b/x-pack/plugins/fleet/common/openapi/bundled.json index cf3ee35fca6df4..727ef4c30f4fdb 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.json +++ b/x-pack/plugins/fleet/common/openapi/bundled.json @@ -7484,8 +7484,7 @@ "type": "string" }, "output_id": { - "type": "string", - "deprecated": true + "type": "string" }, "inputs": { "type": "array", diff --git a/x-pack/plugins/fleet/common/openapi/bundled.yaml b/x-pack/plugins/fleet/common/openapi/bundled.yaml index ad592b9ea18479..7a19615cfe1da3 100644 --- a/x-pack/plugins/fleet/common/openapi/bundled.yaml +++ b/x-pack/plugins/fleet/common/openapi/bundled.yaml @@ -4804,7 +4804,6 @@ components: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml index 29a09174385ec5..956f51a8016f54 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/new_package_policy.yaml @@ -22,7 +22,6 @@ properties: type: string output_id: type: string - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml index 795d92ff1d4a7d..f06a2c9ea49f3a 100644 --- a/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml +++ b/x-pack/plugins/fleet/common/openapi/components/schemas/update_package_policy.yaml @@ -23,8 +23,6 @@ properties: type: string output_id: type: string - description: Not supported output can be set at the agent policy level only - deprecated: true inputs: type: array items: diff --git a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap index 5d747ebd88ebfe..7c549b030a3378 100644 --- a/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap +++ b/x-pack/plugins/fleet/common/services/__snapshots__/simplified_package_policy_helper.test.ts.snap @@ -228,6 +228,7 @@ Object { ], "name": "nginx-1", "namespace": "default", + "output_id": "output123", "package": Object { "name": "nginx", "title": "Nginx", diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts index 0cde9d003dbb19..ff9d6eac7490fb 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.test.ts @@ -37,6 +37,7 @@ describe('toPackagePolicy', () => { namespace: 'default', policy_id: 'policy123', policy_ids: ['policy123'], + output_id: 'output123', description: 'Test description', inputs: { 'nginx-logfile': { diff --git a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts index 98dee1021fe213..8d616bf4cae11a 100644 --- a/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts +++ b/x-pack/plugins/fleet/common/services/simplified_package_policy_helper.ts @@ -43,6 +43,7 @@ export interface SimplifiedPackagePolicy { id?: string; policy_id?: string; policy_ids: string[]; + output_id?: string; namespace: string; name: string; description?: string; @@ -147,6 +148,7 @@ export function simplifiedPackagePolicytoNewPackagePolicy( const { policy_id: policyId, policy_ids: policyIds, + output_id: outputId, namespace, name, description, @@ -161,6 +163,10 @@ export function simplifiedPackagePolicytoNewPackagePolicy( description ); + if (outputId) { + packagePolicy.output_id = outputId; + } + if (packagePolicy.package && options?.experimental_data_stream_features) { packagePolicy.package.experimental_data_stream_features = options.experimental_data_stream_features; diff --git a/x-pack/plugins/fleet/common/types/models/package_policy.ts b/x-pack/plugins/fleet/common/types/models/package_policy.ts index 1b8a407ff8dd92..eb054ade02114a 100644 --- a/x-pack/plugins/fleet/common/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/models/package_policy.ts @@ -81,6 +81,8 @@ export interface NewPackagePolicy { /** @deprecated */ policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; package?: PackagePolicyPackage; inputs: NewPackagePolicyInput[]; vars?: PackagePolicyConfigRecord; diff --git a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts index ff190f0a556cd5..8b74a3142fa973 100644 --- a/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts +++ b/x-pack/plugins/fleet/common/types/rest_spec/package_policy.ts @@ -63,6 +63,7 @@ export type PostDeletePackagePoliciesResponse = Array<{ package?: PackagePolicyPackage; policy_id?: string; policy_ids?: string[]; + output_id?: string; // Support generic errors statusCode?: number; body?: { diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx index 55b8ac90c749c3..c63339d682bb71 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/hooks.tsx @@ -8,6 +8,10 @@ import { useMemo } from 'react'; import { useHistory } from 'react-router-dom'; +import { LICENCE_FOR_OUTPUT_PER_INTEGRATION } from '../../../../../../../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../../../../../../../common/services/output_helpers'; +import { useGetOutputs, useLicense } from '../../../../../../hooks'; + export function useDataStreamId() { const history = useHistory(); @@ -16,3 +20,21 @@ export function useDataStreamId() { return searchParams.get('datastreamId') ?? undefined; }, [history.location.search]); } + +export function useOutputs(packageName: string) { + const licenseService = useLicense(); + const canUseOutputPerIntegration = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + const { data: outputsData, isLoading } = useGetOutputs(); + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const allowedOutputs = useMemo(() => { + if (!outputsData || !canUseOutputPerIntegration) { + return []; + } + return outputsData.items.filter((output) => allowedOutputTypes.includes(output.type)); + }, [allowedOutputTypes, canUseOutputPerIntegration, outputsData]); + return { + isLoading, + canUseOutputPerIntegration, + allowedOutputs, + }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx index c312b48a2d290a..43c3f8092e9218 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/step_define_package_policy.tsx @@ -20,6 +20,7 @@ import { EuiLink, EuiCallOut, EuiSpacer, + EuiSelect, } from '@elastic/eui'; import styled from 'styled-components'; @@ -32,6 +33,7 @@ import { isAdvancedVar } from '../../services'; import type { PackagePolicyValidationResults } from '../../services'; import { PackagePolicyInputVarField } from './components'; +import { useOutputs } from './components/hooks'; // on smaller screens, fields should be displayed in one column const FormGroupResponsiveFields = styled(EuiDescribedFormGroup)` @@ -81,6 +83,14 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ }); } + // Outputs + const { + isLoading: isOutputsLoading, + canUseOutputPerIntegration, + allowedOutputs, + } = useOutputs(packageInfo.name); + + // Managed policy const isManaged = packagePolicy.is_managed; return validationResults ? ( @@ -245,6 +255,7 @@ export const StepDefinePackagePolicy: React.FunctionComponent<{ {isShowingAdvanced ? ( + {/* Namespace */} + + {/* Output */} + {canUseOutputPerIntegration && ( + + + } + helpText={ + + } + > + ({ + value: output.id, + text: output.name, + })), + ]} + value={packagePolicy.output_id || ''} + onChange={(e) => { + updatePackagePolicy({ + output_id: e.target.value.trim() || null, + }); + }} + /> + + + )} + + {/* Data retention settings info */} = ({ return packagePolicy.policy_ids.length || 0; }, []); + const { data: outputsData, isLoading: isOutputsLoading } = useGetOutputs(); + const { output: defaultOutputData } = useDefaultOutput(); + const outputNamesById = useMemo(() => { + const outputs = outputsData?.items ?? []; + return outputs.reduce>((acc, output) => { + acc[output.id] = output.name; + return acc; + }, {}); + }, [outputsData]); + const columns = useMemo( (): EuiInMemoryTableProps['columns'] => [ { @@ -115,6 +127,7 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.nameColumnTitle', { defaultMessage: 'Integration policy', }), + width: '35%', render: (value: string, packagePolicy: InMemoryPackagePolicy) => ( @@ -278,10 +291,67 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ ); }, }, + { + field: 'output_id', + name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.outputColumnTitle', { + defaultMessage: 'Output', + }), + render: (outputId: InMemoryPackagePolicy['output_id']) => { + if (isOutputsLoading) { + return null; + } + if (outputId) { + return {outputNamesById[outputId] || outputId}; + } + if (agentPolicy.data_output_id) { + return ( + <> + + {outputNamesById[agentPolicy.data_output_id] || agentPolicy.data_output_id} + +   + + + ); + } + if (defaultOutputData) { + return ( + <> + + {outputNamesById[defaultOutputData.id] || defaultOutputData.id} + +   + + + ); + } + }, + }, { name: i18n.translate('xpack.fleet.policyDetails.packagePoliciesTable.actionsColumnTitle', { defaultMessage: 'Actions', }), + width: '70px', actions: [ { render: (packagePolicy: InMemoryPackagePolicy) => { @@ -309,8 +379,11 @@ export const PackagePoliciesTable: React.FunctionComponent = ({ agentPolicy, canUseMultipleAgentPolicies, canReadAgentPolicies, - canWriteIntegrationPolicies, getSharedPoliciesNumber, + canWriteIntegrationPolicies, + isOutputsLoading, + defaultOutputData, + outputNamesById, ] ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx index 2a4250bd9707a5..2d8a5b4aa1952a 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/edit_fleet_proxy_flyout/use_fleet_proxy_form.tsx @@ -33,7 +33,7 @@ const ConfirmTitle = () => ( const ConfirmDescription: React.FunctionComponent = ({}) => ( ); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx index 8df533f0206b77..95a94848f47940 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/components/fleet_server_hosts_flyout/use_fleet_server_host_form.test.tsx @@ -11,10 +11,6 @@ import { createFleetTestRendererMock } from '../../../../../../mock'; import { useFleetServerHostsForm } from './use_fleet_server_host_form'; -jest.mock('../../services/agent_and_policies_count', () => ({ - ...jest.requireActual('../../services/agent_and_policies_count'), - getAgentAndPolicyCount: () => ({ agentCount: 0, agentPolicyCount: 0 }), -})); jest.mock('../../hooks/use_confirm_modal', () => ({ ...jest.requireActual('../../hooks/use_confirm_modal'), useConfirmModal: () => ({ confirm: () => true }), diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx index 7e8cdac388cfe5..8720ede4f04b8f 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/settings/services/agent_and_policies_count.tsx @@ -5,17 +5,24 @@ * 2.0. */ -import { sendGetAgentPolicies, sendGetAgents } from '../../../hooks'; +import { sendGetAgentPolicies, sendGetPackagePolicies, sendGetAgents } from '../../../hooks'; import type { Output } from '../../../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE, SO_SEARCH_LIMIT } from '../../../constants'; +import { + AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, + SO_SEARCH_LIMIT, +} from '../../../constants'; export async function getAgentAndPolicyCountForOutput(output: Output) { - let kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + let agentPolicyKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${output.id}" or ${AGENT_POLICY_SAVED_OBJECT_TYPE}.monitoring_output_id:"${output.id}"`; + const packagePolicyKuery = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${output.id}"`; + if (output.is_default) { - kuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; + agentPolicyKuery += ` or (not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*)`; } + const agentPolicies = await sendGetAgentPolicies({ - kuery, + kuery: agentPolicyKuery, page: 1, perPage: SO_SEARCH_LIMIT, }); @@ -23,7 +30,26 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { if (agentPolicies.error) { throw agentPolicies.error; } - const agentPolicyCount = agentPolicies.data?.items?.length ?? 0; + + const packagePolicies = await sendGetPackagePolicies({ + kuery: packagePolicyKuery, + page: 1, + perPage: SO_SEARCH_LIMIT, + }); + + if (packagePolicies.error) { + throw agentPolicies.error; + } + + const agentPolicyIds = (agentPolicies.data?.items || []).map((policy) => policy.id); + const agentPolicyIdsFromPackagePolicies = (packagePolicies.data?.items || []).reduce( + (acc: string[], packagePolicy) => { + return [...acc, ...(packagePolicy.policy_ids || [])]; + }, + [] + ); + const uniqueAgentPolicyIds = new Set([...agentPolicyIds, ...agentPolicyIdsFromPackagePolicies]); + const agentPolicyCount = uniqueAgentPolicyIds.size; let agentCount = 0; if (agentPolicyCount > 0) { @@ -31,7 +57,7 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { page: 1, perPage: 0, // We only need the count here showInactive: false, - kuery: agentPolicies.data?.items.map((policy) => `policy_id:"${policy.id}"`).join(' or '), + kuery: [...uniqueAgentPolicyIds].map((id) => `policy_id:"${id}"`).join(' or '), }); if (agents.error) { @@ -43,28 +69,3 @@ export async function getAgentAndPolicyCountForOutput(output: Output) { return { agentPolicyCount, agentCount }; } - -export async function getAgentAndPolicyCount() { - const agentPolicies = await sendGetAgentPolicies({ - perPage: 0, - }); - - if (agentPolicies.error) { - throw agentPolicies.error; - } - const agentPolicyCount = agentPolicies.data?.total ?? 0; - - const agents = await sendGetAgents({ - page: 1, - perPage: 0, // We only need the count here - showInactive: false, - }); - - if (agents.error) { - throw agents.error; - } - - const agentCount = agents.data?.total ?? 0; - - return { agentPolicyCount, agentCount }; -} diff --git a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts index 2adf814e8ffba8..b85b944c6ee00b 100644 --- a/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts +++ b/x-pack/plugins/fleet/public/hooks/use_multiple_agent_policies.ts @@ -5,12 +5,11 @@ * 2.0. */ +import { LICENCE_FOR_MULTIPLE_AGENT_POLICIES } from '../../common/constants'; import { ExperimentalFeaturesService } from '../services'; import { useLicense } from './use_license'; -export const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - export function useMultipleAgentPolicies() { const licenseService = useLicense(); const { enableReusableIntegrationPolicies } = ExperimentalFeaturesService.get(); diff --git a/x-pack/plugins/fleet/server/errors/index.ts b/x-pack/plugins/fleet/server/errors/index.ts index e2f8b883804841..80d8116baaaa33 100644 --- a/x-pack/plugins/fleet/server/errors/index.ts +++ b/x-pack/plugins/fleet/server/errors/index.ts @@ -65,6 +65,8 @@ export class PackagePolicyNameExistsError extends FleetError {} export class BundledPackageLocationNotFoundError extends FleetError {} export class PackagePolicyRequestError extends FleetError {} +export class PackagePolicyMultipleAgentPoliciesError extends FleetError {} +export class PackagePolicyOutputError extends FleetError {} export class EnrollmentKeyNameExistsError extends FleetError {} export class HostedAgentPolicyRestrictionRelatedError extends FleetError { diff --git a/x-pack/plugins/fleet/server/mocks/index.ts b/x-pack/plugins/fleet/server/mocks/index.ts index 9e517ad928ba3c..9dfb920251e76e 100644 --- a/x-pack/plugins/fleet/server/mocks/index.ts +++ b/x-pack/plugins/fleet/server/mocks/index.ts @@ -182,6 +182,7 @@ export const createPackagePolicyServiceMock = (): jest.Mocked; @@ -202,6 +201,7 @@ describe('When calling package policy', () => { }; beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); // @ts-ignore const postMock = routerMock.versioned.post.mock; // @ts-ignore @@ -221,29 +221,6 @@ describe('When calling package policy', () => { }, }); }); - - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - const request = getCreateKibanaRequest({ ...newPolicy, policy_ids: ['1', '2'] } as any); - await createPackagePolicyHandler(context, request as any, response); - expect(response.customError).not.toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); }); describe('Update api handler', () => { @@ -308,6 +285,7 @@ describe('When calling package policy', () => { }); beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); packagePolicyServiceMock.update.mockImplementation((soClient, esClient, policyId, newData) => Promise.resolve(newData as PackagePolicy) ); @@ -393,58 +371,6 @@ describe('When calling package policy', () => { }); }); - it('should throw if no enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are only available with an Enterprise license', - }, - }); - }); - - it('should not throw if enterprise license and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.ok).toHaveBeenCalled(); - }); - - it('should throw if enterprise license and feature flag is disabled and multiple policy_ids is provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); - const request = getUpdateKibanaRequest({ policy_ids: ['1', '2'] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'Reusable integration policies are not supported', - }, - }); - }); - - it('should throw if empty policy_ids are provided', async () => { - jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); - jest - .spyOn(appContextService, 'getExperimentalFeatures') - .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); - const request = getUpdateKibanaRequest({ policy_ids: [] } as any); - await routeHandler(context, request, response); - expect(response.customError).toHaveBeenCalledWith({ - statusCode: 400, - body: { - message: 'At least one agent policy id must be provided', - }, - }); - }); - it('should rename the agentless agent policy to sync with the package policy name if agentless is enabled', async () => { jest.spyOn(appContextService, 'getCloud').mockReturnValue({ isCloudEnabled: true } as any); jest diff --git a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts index 24872b2db72f20..31e38125dead5d 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/handlers.ts @@ -63,7 +63,6 @@ import { import type { SimplifiedPackagePolicy } from '../../../common/services/simplified_package_policy_helper'; import { - canUseMultipleAgentPolicies, isSimplifiedCreatePackagePolicyRequest, removeFieldsFromInputSchema, renameAgentlessAgentPolicy, @@ -242,21 +241,12 @@ export const createPackagePolicyHandler: FleetRequestHandler< const authorizationHeader = HTTPAuthorizationHeader.parseFromRequest(request, user?.username); let wasPackageAlreadyInstalled = false; - if ('output_id' in newPolicy) { - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - delete newPolicy.output_id; - } const spaceId = fleetContext.spaceId; try { if (!newPolicy.policy_id && (!newPolicy.policy_ids || newPolicy.policy_ids.length === 0)) { throw new PackagePolicyRequestError('Either policy_id or policy_ids must be provided'); } - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newPolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } - let newPackagePolicy: NewPackagePolicy; if (isSimplifiedCreatePackagePolicyRequest(newPolicy)) { if (!pkg) { @@ -369,11 +359,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< try { const { force, package: pkg, ...body } = request.body; - // TODO Remove deprecated APIs https://github.com/elastic/kibana/issues/121485 - if ('output_id' in body) { - delete body.output_id; - } - let newData: NewPackagePolicy; if ( @@ -420,10 +405,6 @@ export const updatePackagePolicyHandler: FleetRequestHandler< } } newData.inputs = alignInputsAndStreams(newData.inputs); - const { canUseReusablePolicies, errorMessage } = canUseMultipleAgentPolicies(); - if ((newData.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { - throw new PackagePolicyRequestError(errorMessage); - } if (newData.policy_ids && newData.policy_ids.length === 0) { throw new PackagePolicyRequestError('At least one agent policy id must be provided'); diff --git a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts index f2ffc641c36ab5..032dab4a07acc7 100644 --- a/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts +++ b/x-pack/plugins/fleet/server/routes/package_policy/utils/index.ts @@ -21,8 +21,7 @@ import type { PackagePolicyInput, NewPackagePolicyInput, } from '../../../types'; -import { appContextService } from '../../../services'; -import { agentPolicyService, licenseService } from '../../../services'; +import { agentPolicyService } from '../../../services'; import type { SimplifiedPackagePolicy } from '../../../../common/services/simplified_package_policy_helper'; import { PackagePolicyRequestError } from '../../../errors'; import type { NewPackagePolicyInputStream } from '../../../../common'; @@ -56,20 +55,6 @@ export function removeFieldsFromInputSchema( }); } -const LICENCE_FOR_MULTIPLE_AGENT_POLICIES = 'enterprise'; - -export function canUseMultipleAgentPolicies() { - const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); - const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); - - return { - canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, - errorMessage: !hasEnterpriseLicence - ? 'Reusable integration policies are only available with an Enterprise license' - : 'Reusable integration policies are not supported', - }; -} - /** * If an agentless agent policy is associated with the package policy, * it will rename the agentless agent policy of a package policy to keep it in sync with the package policy name. diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 4b76a952a0f3b4..f6fcae4de6505c 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -481,6 +481,7 @@ export const getSavedObjectTypes = ( is_managed: { type: 'boolean' }, policy_id: { type: 'keyword' }, policy_ids: { type: 'keyword' }, + output_id: { type: 'keyword' }, package: { properties: { name: { type: 'keyword' }, @@ -639,6 +640,16 @@ export const getSavedObjectTypes = ( }, ], }, + '14': { + changes: [ + { + type: 'mappings_addition', + addedMappings: { + output_id: { type: 'keyword' }, + }, + }, + ], + }, }, migrations: { '7.10.0': migratePackagePolicyToV7100, diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts index cf835ec87c949a..d7e73d67a3c604 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_5_0.ts @@ -14,7 +14,6 @@ export const migratePackagePolicyToV850: SavedObjectMigrationFn { - // @ts-expect-error output_id property does not exists anymore delete packagePolicyDoc.attributes.output_id; return packagePolicyDoc; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap index 86f8126c1c45ae..8518e43fdaf0d2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap +++ b/x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap @@ -1,5 +1,173 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`getFullAgentPolicy should return the right outputs and permissions when package policies use their own outputs 1`] = ` +Object { + "agent": Object { + "download": Object { + "sourceURI": "http://default-registry.co", + }, + "features": Object {}, + "monitoring": Object { + "enabled": false, + "logs": false, + "metrics": false, + }, + "protection": Object { + "enabled": false, + "signing_key": "", + "uninstall_token_hash": "", + }, + }, + "fleet": Object { + "hosts": Array [ + "http://fleetserver:8220", + ], + }, + "id": "integration-output-policy", + "inputs": Array [ + Object { + "data_stream": Object { + "namespace": "policyspace", + }, + "id": "test-logs-package-policy-using-output", + "meta": Object { + "package": Object { + "name": "test_package", + "version": "0.0.0", + }, + }, + "name": "test-policy-1", + "package_policy_id": "package-policy-using-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "test-remote-id", + }, + Object { + "data_stream": Object { + "namespace": "defaultspace", + }, + "id": "test-logs-package-policy-no-output", + "meta": Object { + "package": Object { + "name": "system", + "version": "1.0.0", + }, + }, + "name": "test-policy-2", + "package_policy_id": "package-policy-no-output", + "revision": 1, + "streams": Array [ + Object { + "data_stream": Object { + "dataset": "some-logs", + "type": "logs", + }, + "id": "test-logs", + }, + ], + "type": "test-logs", + "use_output": "data-output-id", + }, + ], + "output_permissions": Object { + "data-output-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-no-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-defaultspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + "default": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "_elastic_agent_monitoring": Object { + "indices": Array [ + Object { + "names": Array [], + "privileges": Array [], + }, + ], + }, + }, + "test-remote-id": Object { + "_elastic_agent_checks": Object { + "cluster": Array [ + "monitor", + ], + }, + "package-policy-using-output": Object { + "indices": Array [ + Object { + "names": Array [ + "logs-some-logs-policyspace", + ], + "privileges": Array [ + "auto_configure", + "create_doc", + ], + }, + ], + }, + }, + }, + "outputs": Object { + "data-output-id": Object { + "hosts": Array [ + "http://es-data.co:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "default": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "type": "elasticsearch", + }, + "test-remote-id": Object { + "hosts": Array [ + "http://127.0.0.1:9201", + ], + "preset": "balanced", + "service_token": undefined, + "type": "remote_elasticsearch", + }, + }, + "revision": 1, + "secret_references": Array [], + "signed": Object { + "data": "", + "signature": "", + }, +} +`; + exports[`getFullAgentPolicy should support a different data output 1`] = ` Object { "agent": Object { diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts index 5701a60b56d037..6a084b5dde5865 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts @@ -401,6 +401,126 @@ describe('getFullAgentPolicy', () => { expect(agentPolicy?.outputs['test-remote-id']).toBeDefined(); }); + it('should return the right outputs and permissions when package policies use their own outputs', async () => { + mockedGetPackageInfo.mockResolvedValue({ + data_streams: [ + { + type: 'logs', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.metricbeat', + }, + { + type: 'logs', + dataset: 'elastic_agent.filebeat', + }, + { + type: 'metrics', + dataset: 'elastic_agent.filebeat', + }, + ], + } as PackageInfo); + mockAgentPolicy({ + id: 'integration-output-policy', + status: 'active', + package_policies: [ + { + id: 'package-policy-using-output', + name: 'test-policy-1', + namespace: 'policyspace', + enabled: true, + package: { name: 'test_package', version: '0.0.0', title: 'Test Package' }, + output_id: 'test-remote-id', + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + { + id: 'package-policy-no-output', + name: 'test-policy-2', + namespace: '', + enabled: true, + package: { name: 'system', version: '1.0.0', title: 'System' }, + inputs: [ + { + type: 'test-logs', + enabled: true, + streams: [ + { + id: 'test-logs', + enabled: true, + data_stream: { type: 'logs', dataset: 'some-logs' }, + }, + ], + }, + { + type: 'test-metrics', + enabled: false, + streams: [ + { + id: 'test-logs', + enabled: false, + data_stream: { type: 'metrics', dataset: 'some-metrics' }, + }, + ], + }, + ], + created_at: '', + updated_at: '', + created_by: '', + updated_by: '', + revision: 1, + policy_id: '', + policy_ids: [''], + }, + ], + is_managed: false, + namespace: 'defaultspace', + revision: 1, + name: 'Policy', + updated_at: '2020-01-01', + updated_by: 'qwerty', + is_protected: false, + data_output_id: 'data-output-id', + }); + + const agentPolicy = await getFullAgentPolicy( + savedObjectsClientMock.create(), + 'integration-output-policy' + ); + expect(agentPolicy).toMatchSnapshot(); + }); + it('should return the sourceURI from the agent policy', async () => { mockAgentPolicy({ namespace: 'default', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts index efc3a732149d64..0cfce9eb264f45 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts @@ -196,16 +196,46 @@ export async function getFullAgentPolicy( fullAgentPolicy.namespaces = [agentPolicy.space_id]; } - const dataPermissions = - (await storedPackagePoliciesToAgentPermissions( + const packagePoliciesByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = []; + return acc; + }, + {} + ); + (agentPolicy.package_policies || []).forEach((packagePolicy) => { + const packagePolicyDataOutput = packagePolicy.output_id + ? outputs.find((output) => output.id === packagePolicy.output_id) + : undefined; + if (packagePolicyDataOutput) { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(packagePolicyDataOutput)].push( + packagePolicy + ); + } else { + packagePoliciesByOutputId[getOutputIdForAgentPolicy(dataOutput)].push(packagePolicy); + } + }); + + const dataPermissionsByOutputId = Object.keys(fullAgentPolicy.outputs).reduce( + (acc: Record, outputId) => { + acc[outputId] = {}; + return acc; + }, + {} + ); + for (const [outputId, packagePolicies] of Object.entries(packagePoliciesByOutputId)) { + const dataPermissions = await storedPackagePoliciesToAgentPermissions( packageInfoCache, agentPolicy.namespace, - agentPolicy.package_policies - )) || {}; - - dataPermissions._elastic_agent_checks = { - cluster: DEFAULT_CLUSTER_PERMISSIONS, - }; + packagePolicies + ); + dataPermissionsByOutputId[outputId] = { + _elastic_agent_checks: { + cluster: DEFAULT_CLUSTER_PERMISSIONS, + }, + ...(dataPermissions || {}), + }; + } const monitoringPermissions = await getMonitoringPermissions( soClient, @@ -233,8 +263,11 @@ export async function getFullAgentPolicy( Object.assign(permissions, monitoringPermissions); } - if (outputId === getOutputIdForAgentPolicy(dataOutput)) { - Object.assign(permissions, dataPermissions); + if ( + outputId === getOutputIdForAgentPolicy(dataOutput) || + packagePoliciesByOutputId[outputId].length > 0 + ) { + Object.assign(permissions, dataPermissionsByOutputId[outputId]); } outputPermissions[outputId] = permissions; diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index dfdcf26adaa4ed..0afef0170a99f1 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -13,7 +13,7 @@ import { appContextService } from '..'; import { outputService } from '../output'; import { validateOutputForPolicy } from '.'; -import { validateOutputForNewPackagePolicy } from './outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './outputs_helpers'; jest.mock('../app_context'); jest.mock('../output'); @@ -254,14 +254,14 @@ describe('validateOutputForPolicy', () => { }); }); -describe('validateOutputForNewPackagePolicy', () => { - it('should not allow fleet_server integration to be added to a policy using a logstash output', async () => { +describe('validateAgentPolicyOutputForIntegration', () => { + it('should not allow fleet_server integration to be added or edited to a policy using a logstash output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'logstash', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -273,15 +273,29 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "fleet_server" cannot be added to agent policy "Agent policy" because it uses output type "logstash".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'fleet_server', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "logstash" which cannot be used for integration "fleet_server".' + ); }); - it('should not allow apm integration to be added to a policy using a kafka output', async () => { + it('should not allow apm integration to be added or edited to a policy using a kafka output', async () => { mockHasLicence(true); mockedOutputService.get.mockResolvedValue({ type: 'kafka', } as any); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -293,6 +307,20 @@ describe('validateOutputForNewPackagePolicy', () => { ).rejects.toThrow( 'Integration "apm" cannot be added to agent policy "Agent policy" because it uses output type "kafka".' ); + await expect( + validateAgentPolicyOutputForIntegration( + savedObjectsClientMock.create(), + { + name: 'Agent policy', + data_output_id: 'test1', + monitoring_output_id: 'test1', + } as any, + 'apm', + false + ) + ).rejects.toThrow( + 'Agent policy "Agent policy" uses output type "kafka" which cannot be used for integration "apm".' + ); }); it('should not allow synthetics integration to be added to a policy using a default logstash output', async () => { @@ -302,7 +330,7 @@ describe('validateOutputForNewPackagePolicy', () => { } as any); mockedOutputService.getDefaultDataOutputId.mockResolvedValue('default'); await expect( - validateOutputForNewPackagePolicy( + validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -320,7 +348,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'logstash', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', @@ -335,7 +363,7 @@ describe('validateOutputForNewPackagePolicy', () => { type: 'elasticsearch', } as any); - await validateOutputForNewPackagePolicy( + await validateAgentPolicyOutputForIntegration( savedObjectsClientMock.create(), { name: 'Agent policy', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts index 67f5a7772aa52a..66f45427601e6e 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/outputs_helpers.ts @@ -96,10 +96,11 @@ export async function validateOutputForPolicy( } } -export async function validateOutputForNewPackagePolicy( +export async function validateAgentPolicyOutputForIntegration( soClient: SavedObjectsClientContract, agentPolicy: AgentPolicy, - packageName: string + packageName: string, + isNewPackagePolicy: boolean = true ) { const allowedOutputTypeForPolicy = getAllowedOutputTypesForIntegration(packageName); @@ -109,9 +110,15 @@ export async function validateOutputForNewPackagePolicy( if (isOutputTypeRestricted) { const dataOutput = await getDataOutputForAgentPolicy(soClient, agentPolicy); if (!allowedOutputTypeForPolicy.includes(dataOutput.type)) { - throw new OutputInvalidError( - `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` - ); + if (isNewPackagePolicy) { + throw new OutputInvalidError( + `Integration "${packageName}" cannot be added to agent policy "${agentPolicy.name}" because it uses output type "${dataOutput.type}".` + ); + } else { + throw new OutputInvalidError( + `Agent policy "${agentPolicy.name}" uses output type "${dataOutput.type}" which cannot be used for integration "${packageName}".` + ); + } } } } diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts index 40cda7583a3c15..9004d544f399db 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.test.ts @@ -222,6 +222,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { version: '0.0.0', }, inputs: [mockInput, mockInput2], + output_id: 'new-output', }, { ...mockPackagePolicy, @@ -243,7 +244,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-logs', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', @@ -270,7 +271,7 @@ describe('Fleet - storedPackagePoliciesToAgentInputs', () => { revision: 1, type: 'test-metrics', data_stream: { namespace: 'default' }, - use_output: 'default', + use_output: 'new-output', meta: { package: { name: 'mock_package', diff --git a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts index d7f0c70a0786b7..807312fe5e7cb2 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/package_policies_to_agent_inputs.ts @@ -27,7 +27,7 @@ const isPolicyEnabled = (packagePolicy: PackagePolicy) => { export const storedPackagePolicyToAgentInputs = ( packagePolicy: PackagePolicy, packageInfo?: PackageInfo, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, addFields?: FullAgentPolicyAddFields ): FullAgentPolicyInput[] => { @@ -62,7 +62,7 @@ export const storedPackagePolicyToAgentInputs = ( data_stream: { namespace: packagePolicy?.namespace || agentPolicyNamespace || 'default', // custom namespace has precedence on agent policy's one }, - use_output: outputId, + use_output: packagePolicy.output_id || agentPolicyOutputId, package_policy_id: packagePolicy.id, ...getFullInputStreams(input), }; @@ -140,7 +140,7 @@ export const getFullInputStreams = ( export const storedPackagePoliciesToAgentInputs = async ( packagePolicies: PackagePolicy[], packageInfoCache: Map, - outputId: string = DEFAULT_OUTPUT.name, + agentPolicyOutputId: string = DEFAULT_OUTPUT.name, agentPolicyNamespace?: string, globalDataTags?: GlobalDataTag[] ): Promise => { @@ -164,7 +164,7 @@ export const storedPackagePoliciesToAgentInputs = async ( ...storedPackagePolicyToAgentInputs( packagePolicy, packageInfo, - outputId, + agentPolicyOutputId, agentPolicyNamespace, addFields ) diff --git a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts index 0108e9cd97721c..52dd34a757693b 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/related_saved_objects.ts @@ -35,11 +35,20 @@ export async function fetchRelatedSavedObjects( const monitoringOutputId = agentPolicy.monitoring_output_id || defaultMonitoringOutputId || dataOutputId; + const outputIds = uniq([ + dataOutputId, + monitoringOutputId, + ...(agentPolicy.package_policies || []).reduce((acc: string[], packagePolicy) => { + if (packagePolicy.output_id) { + acc.push(packagePolicy.output_id); + } + return acc; + }, []), + ]); + const [outputs, { host: downloadSourceUri, proxy_id: downloadSourceProxyId }, fleetServerHosts] = await Promise.all([ - outputService.bulkGet(soClient, uniq([dataOutputId, monitoringOutputId]), { - ignoreNotFound: true, - }), + outputService.bulkGet(soClient, outputIds, { ignoreNotFound: true }), getSourceUriForAgentPolicy(soClient, agentPolicy), getFleetServerHostsForAgentPolicy(soClient, agentPolicy).catch((err) => { appContextService diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index a5dbbc6b233b3b..2c2c747681887f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -23,7 +23,7 @@ import type { NewAgentPolicy, PreconfiguredAgentPolicy, } from '../types'; -import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; +import { AGENT_POLICY_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { AGENT_POLICY_INDEX, SO_SEARCH_LIMIT } from '../../common'; @@ -43,32 +43,63 @@ import type { UninstallTokenServiceInterface } from './security/uninstall_token_ function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); + const mockPolicy = { + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes: agentPolicyAttributes as AgentPolicy, + }; mock.get.mockImplementation(async (type: string, id: string) => { return { - type, id, - references: [], - attributes: agentPolicyAttributes as AgentPolicy, + ...mockPolicy, }; }); - mock.find.mockImplementation(async (options) => { + mock.bulkGet.mockImplementation(async (options) => { return { - saved_objects: [ - { - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - attributes: { - fleet_server_hosts: ['http://fleetserver:8220'], - }, - type: 'ingest_manager_settings', - score: 1, - references: [], - }, - ], - total: 1, - page: 1, - per_page: 1, + saved_objects: [], }; }); + mock.find.mockImplementation(async (options) => { + switch (options.type) { + case AGENT_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [ + { + id: 'agent-policy-id', + score: 1, + ...mockPolicy, + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + case PACKAGE_POLICY_SAVED_OBJECT_TYPE: + return { + saved_objects: [], + total: 0, + page: 1, + per_page: 1, + }; + default: + return { + saved_objects: [ + { + id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', + attributes: { + fleet_server_hosts: ['http://fleetserver:8220'], + }, + type: 'ingest_manager_settings', + score: 1, + references: [], + }, + ], + total: 1, + page: 1, + per_page: 1, + }; + } + }); return mock; } @@ -754,12 +785,12 @@ describe('Agent policy', () => { [ expect.objectContaining({ attributes: expect.objectContaining({ - fleet_server_hosts: ['http://fleetserver:8220'], - revision: NaN, - updated_by: 'system', + monitoring_enabled: ['metrics'], }), - id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c', - type: 'ingest_manager_settings', + id: 'agent-policy-id', + namespace: undefined, + type: 'ingest-agent-policies', + version: undefined, }), ], expect.objectContaining({ diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index d243ef8b60e168..42bede95986b6c 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -16,7 +16,7 @@ import type { SavedObjectsBulkUpdateObject, SavedObjectsBulkUpdateResponse, SavedObjectsClientContract, - SavedObjectsFindResult, + SavedObject, SavedObjectsUpdateResponse, } from '@kbn/core/server'; import { SavedObjectsUtils } from '@kbn/core/server'; @@ -41,6 +41,7 @@ import { import type { HTTPAuthorizationHeader } from '../../common/http_authorization_header'; import { + PACKAGE_POLICY_SAVED_OBJECT_TYPE, AGENT_POLICY_SAVED_OBJECT_TYPE, AGENTS_PREFIX, FLEET_AGENT_POLICIES_SCHEMA_VERSION, @@ -56,6 +57,7 @@ import type { NewAgentPolicy, NewPackagePolicy, PackagePolicy, + PackagePolicySOAttributes, PostAgentPolicyCreateCallback, PostAgentPolicyUpdateCallback, PreconfiguredAgentPolicy, @@ -876,7 +878,7 @@ class AgentPolicyService { private async _bumpPolicies( internalSoClientWithoutSpaceExtension: SavedObjectsClientContract, esClient: ElasticsearchClient, - savedObjectsResults: Array>, + savedObjectsResults: Array>, options?: { user?: AuthenticatedUser } ): Promise> { const bumpedPolicies = savedObjectsResults.map( @@ -938,10 +940,12 @@ class AgentPolicyService { outputId: string, options?: { user?: AuthenticatedUser } ): Promise> { + const { useSpaceAwareness } = appContextService.getExperimentalFeatures(); const internalSoClientWithoutSpaceExtension = appContextService.getInternalUserSOClientWithoutSpaceExtension(); - const currentPolicies = + // All agent policies directly using output + const agentPoliciesUsingOutput = await internalSoClientWithoutSpaceExtension.find({ type: SAVED_OBJECT_TYPE, fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], @@ -950,10 +954,47 @@ class AgentPolicyService { perPage: SO_SEARCH_LIMIT, namespaces: ['*'], }); + + // All package policies directly using output + const packagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.find({ + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + fields: ['output_id', 'namespaces', 'policy_ids'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + namespaces: ['*'], + }); + + const agentPolicyIdsDirectlyUsingOutput = agentPoliciesUsingOutput.saved_objects.map( + (agentPolicySO) => agentPolicySO.id + ); + const agentPolicyIdsOfPackagePoliciesUsingOutput = + packagePoliciesUsingOutput.saved_objects.reduce((acc: Set, packagePolicySO) => { + const newIds = packagePolicySO.attributes.policy_ids.filter((policyId) => { + return !agentPolicyIdsDirectlyUsingOutput.includes(policyId); + }); + return new Set([...acc, ...newIds]); + }, new Set()); + + // Agent policies of the identified package policies, excluding ones already retrieved directly + const agentPoliciesOfPackagePoliciesUsingOutput = + await internalSoClientWithoutSpaceExtension.bulkGet( + [...agentPolicyIdsOfPackagePoliciesUsingOutput].map((id) => ({ + type: SAVED_OBJECT_TYPE, + id, + fields: ['revision', 'data_output_id', 'monitoring_output_id', 'namespaces'], + ...(useSpaceAwareness ? { namespaces: ['*'] } : {}), + })) + ); + return this._bumpPolicies( internalSoClientWithoutSpaceExtension, esClient, - currentPolicies.saved_objects, + [ + ...agentPoliciesUsingOutput.saved_objects, + ...agentPoliciesOfPackagePoliciesUsingOutput.saved_objects, + ], options ); } diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 3bc9003162f448..0951657fae6dd0 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -19,10 +19,12 @@ import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { auditLoggingService } from './audit_logging'; jest.mock('./app_context'); jest.mock('./agent_policy'); +jest.mock('./package_policy'); jest.mock('./audit_logging'); const mockedAuditLoggingService = auditLoggingService as jest.Mocked; @@ -43,6 +45,7 @@ mockedAppContextService.getLogger.mockImplementation(() => { mockedAppContextService.getExperimentalFeatures.mockReturnValue({} as any); const mockedAgentPolicyService = agentPolicyService as jest.Mocked; +const mockedPackagePolicyService = packagePolicyService as jest.Mocked; const CLOUD_ID = 'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw=='; @@ -219,6 +222,7 @@ function getMockedSoClient( }); mockedAppContextService.getInternalUserSOClient.mockReturnValue(soClient); + mockedAppContextService.getInternalUserSOClientWithoutSpaceExtension.mockReturnValue(soClient); return soClient; } @@ -288,17 +292,23 @@ describe('Output Service', () => { } as unknown as ReturnType; beforeEach(() => { + mockedAgentPolicyService.getByIDs.mockResolvedValue([]); mockedAgentPolicyService.list.mockClear(); mockedAgentPolicyService.hasAPMIntegration.mockClear(); mockedAgentPolicyService.hasFleetServerIntegration.mockClear(); mockedAgentPolicyService.hasSyntheticsIntegration.mockClear(); mockedAgentPolicyService.removeOutputFromAll.mockReset(); + mockedPackagePolicyService.removeOutputFromAll.mockReset(); mockedAppContextService.getInternalUserSOClient.mockReset(); mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); mockedAgentPolicyService.update.mockReset(); }); + afterEach(() => { + mockedAgentPolicyService.getByIDs.mockClear(); + }); + describe('create', () => { describe('elasticsearch output', () => { it('works with a predefined id', async () => { @@ -1412,7 +1422,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new Elasticsearch output.' ); }); @@ -1428,7 +1438,7 @@ describe('Output Service', () => { hosts: ['test:4343'], }) ).rejects.toThrowError( - 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Synthetics integration in synthetics policy. Please create a new Elasticsearch output.' ); }); @@ -1758,6 +1768,7 @@ describe('Output Service', () => { fromPreconfiguration: true, }); expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled(); + expect(mockedPackagePolicyService.removeOutputFromAll).toBeCalled(); expect(soClient.delete).toBeCalled(); }); diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index f42867f0686bb9..2748ad78e765b9 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -38,6 +38,7 @@ import type { } from '../types'; import { AGENT_POLICY_SAVED_OBJECT_TYPE, + PACKAGE_POLICY_SAVED_OBJECT_TYPE, DEFAULT_OUTPUT, DEFAULT_OUTPUT_ID, OUTPUT_SAVED_OBJECT_TYPE, @@ -63,6 +64,7 @@ import { import type { OutputType } from '../types'; import { agentPolicyService } from './agent_policy'; +import { packagePolicyService } from './package_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; import { auditLoggingService } from './audit_logging'; @@ -124,40 +126,63 @@ function outputSavedObjectToOutput(so: SavedObject): Output }; } -async function getAgentPoliciesPerOutput( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - let kuery: string; +async function getAgentPoliciesPerOutput(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + let agentPoliciesKuery: string; + const packagePoliciesKuery: string = `${PACKAGE_POLICY_SAVED_OBJECT_TYPE}.output_id:"${outputId}"`; if (outputId) { if (isDefault) { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}" or not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { - kuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; + agentPoliciesKuery = `${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:"${outputId}"`; } } else { if (isDefault) { - kuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; + agentPoliciesKuery = `not ${AGENT_POLICY_SAVED_OBJECT_TYPE}.data_output_id:*`; } else { return; } } - const agentPolicySO = await agentPolicyService.list(soClient, { - kuery, + // Get agent policies directly using output + const directAgentPolicies = await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: agentPoliciesKuery, perPage: SO_SEARCH_LIMIT, withPackagePolicies: true, }); - return agentPolicySO?.items; + const directAgentPolicyIds = directAgentPolicies?.items.map((policy) => policy.id); + + // Get package policies using output and derive agent policies from that which + // are not already identfied above. The IDs cannot be used as part of the kuery + // above since the underlying saved object client .find() only filters on attributes + const packagePolicySOs = await packagePolicyService.list(internalSoClientWithoutSpaceExtension, { + kuery: packagePoliciesKuery, + perPage: SO_SEARCH_LIMIT, + }); + const agentPolicyIdsFromPackagePolicies = [ + ...new Set( + packagePolicySOs?.items.reduce((acc: string[], packagePolicy) => { + return [ + ...acc, + ...packagePolicy.policy_ids.filter((id) => !directAgentPolicyIds?.includes(id)), + ]; + }, []) + ), + ]; + const agentPoliciesFromPackagePolicies = await agentPolicyService.getByIDs( + internalSoClientWithoutSpaceExtension, + agentPolicyIdsFromPackagePolicies, + { + withPackagePolicies: true, + } + ); + + return [...directAgentPolicies.items, ...agentPoliciesFromPackagePolicies]; } -async function validateLogstashOutputNotUsedInAPMPolicy( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { - const agentPolicies = await getAgentPoliciesPerOutput(soClient, outputId, isDefault); +async function validateLogstashOutputNotUsedInAPMPolicy(outputId?: string, isDefault?: boolean) { + const agentPolicies = await getAgentPoliciesPerOutput(outputId, isDefault); // Validate no policy with APM use that policy if (agentPolicies) { @@ -169,16 +194,19 @@ async function validateLogstashOutputNotUsedInAPMPolicy( } } -async function findPoliciesWithFleetServerOrSynthetics( - soClient: SavedObjectsClientContract, - outputId?: string, - isDefault?: boolean -) { +async function findPoliciesWithFleetServerOrSynthetics(outputId?: string, isDefault?: boolean) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + // find agent policies by outputId // otherwise query all the policies const agentPolicies = outputId - ? await getAgentPoliciesPerOutput(soClient, outputId, isDefault) - : (await agentPolicyService.list(soClient, { withPackagePolicies: true }))?.items; + ? await getAgentPoliciesPerOutput(outputId, isDefault) + : ( + await agentPolicyService.list(internalSoClientWithoutSpaceExtension, { + withPackagePolicies: true, + }) + )?.items; const policiesWithFleetServer = agentPolicies?.filter((policy) => agentPolicyService.hasFleetServerIntegration(policy)) || []; @@ -199,13 +227,12 @@ function validateOutputNotUsedInPolicy( dataOutputType )} output cannot be used with ${integrationName} integration in ${ agentPolicy.name - }. Please create a new ElasticSearch output.` + }. Please create a new Elasticsearch output.` ); } } async function validateTypeChanges( - soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, id: string, data: Nullable>, @@ -213,12 +240,14 @@ async function validateTypeChanges( defaultDataOutputId: string | null, fromPreconfiguration: boolean ) { + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); const mergedIsDefault = data.is_default ?? originalOutput.is_default; const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient, id, mergedIsDefault); + await findPoliciesWithFleetServerOrSynthetics(id, mergedIsDefault); if (data.type === outputType.Logstash || originalOutput.type === outputType.Logstash) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, id, mergedIsDefault); + await validateLogstashOutputNotUsedInAPMPolicy(id, mergedIsDefault); } // prevent changing an ES output to logstash or kafka if it's used by fleet server or synthetics policies if ( @@ -230,7 +259,7 @@ async function validateTypeChanges( validateOutputNotUsedInPolicy(policiesWithSynthetics, data.type, 'Synthetics'); } await updateAgentPoliciesDataOutputId( - soClient, + internalSoClientWithoutSpaceExtension, esClient, data, mergedIsDefault, @@ -274,7 +303,7 @@ class OutputService { return appContextService.getInternalUserSOClient(fakeRequest); } - private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { + private async _getDefaultDataOutputsSO() { const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], @@ -403,7 +432,7 @@ class OutputService { } public async getDefaultDataOutputId(soClient: SavedObjectsClientContract) { - const outputs = await this._getDefaultDataOutputsSO(soClient); + const outputs = await this._getDefaultDataOutputsSO(); if (!outputs.saved_objects.length) { return null; @@ -454,7 +483,7 @@ class OutputService { const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); if (output.type === outputType.Logstash || output.type === outputType.Kafka) { - await validateLogstashOutputNotUsedInAPMPolicy(soClient, undefined, data.is_default); + await validateLogstashOutputNotUsedInAPMPolicy(undefined, data.is_default); if (!appContextService.getEncryptedSavedObjectsSetup()?.canEncrypt) { throw new FleetEncryptedSavedObjectEncryptionKeyRequired( `${output.type} output needs encrypted saved object api key to be set` @@ -462,7 +491,7 @@ class OutputService { } } const { policiesWithFleetServer, policiesWithSynthetics } = - await findPoliciesWithFleetServerOrSynthetics(soClient); + await findPoliciesWithFleetServerOrSynthetics(); await updateAgentPoliciesDataOutputId( soClient, esClient, @@ -740,8 +769,17 @@ class OutputService { throw new OutputUnauthorizedError(`Default monitoring output ${id} cannot be deleted.`); } + const internalSoClientWithoutSpaceExtension = + appContextService.getInternalUserSOClientWithoutSpaceExtension(); + + await packagePolicyService.removeOutputFromAll( + internalSoClientWithoutSpaceExtension, + appContextService.getInternalUserESClient(), + id + ); + await agentPolicyService.removeOutputFromAll( - soClient, + internalSoClientWithoutSpaceExtension, appContextService.getInternalUserESClient(), id ); @@ -808,7 +846,6 @@ class OutputService { const mergedType = data.type ?? originalOutput.type; const defaultDataOutputId = await this.getDefaultDataOutputId(soClient); await validateTypeChanges( - soClient, esClient, id, updateData, diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts index 9bf58298b09d35..311bfeb339f20f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.test.ts @@ -4,10 +4,15 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; import { PackagePolicyMocks } from '../../mocks'; -import { mapPackagePolicySavedObjectToPackagePolicy } from './utils'; +import { appContextService } from '../app_context'; +import { licenseService } from '../license'; +import { outputService } from '../output'; + +import { mapPackagePolicySavedObjectToPackagePolicy, preflightCheckPackagePolicy } from './utils'; describe('Package Policy Utils', () => { describe('mapPackagePolicySavedObjectToPackagePolicy()', () => { @@ -38,6 +43,7 @@ describe('Package Policy Utils', () => { }, policy_id: '444-555-666', policy_ids: ['444-555-666'], + output_id: 'output-123', revision: 1, secret_references: [], updated_at: '2024-01-25T15:21:13.389Z', @@ -47,4 +53,94 @@ describe('Package Policy Utils', () => { }); }); }); + + describe('preflightCheckPackagePolicy', () => { + beforeEach(() => { + jest.spyOn(licenseService, 'hasAtLeast').mockClear(); + jest.spyOn(appContextService, 'getExperimentalFeatures').mockClear(); + }); + const soClient = savedObjectsClientMock.create(); + const testPolicy = { + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_ids: ['test'], + inputs: [], + package: { + name: 'test', + title: 'Test', + version: '0.0.1', + }, + }; + + it('should throw if no enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError( + 'Reusable integration policies are only available with an Enterprise license' + ); + }); + + it('should throw if enterprise license and multiple policy_ids is provided but no feature flag', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: false } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).rejects.toThrowError('Reusable integration policies are not supported'); + }); + + it('should not throw if enterprise license and multiple policy_ids is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(appContextService, 'getExperimentalFeatures') + .mockReturnValue({ enableReusableIntegrationPolicies: true } as any); + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, policy_ids: ['1', '2'] }) + ).resolves.not.toThrow(); + }); + + it('should throw if no valid license and output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(false); + + await expect( + preflightCheckPackagePolicy(soClient, { ...testPolicy, output_id: 'some-output' }) + ).rejects.toThrowError('Output per integration is only available with an enterprise license'); + }); + + it('should throw if valid license and an incompatible output_id for the package is given', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'non-es-output', type: 'kafka' } as any); + + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'non-es-output', + package: { name: 'apm', version: '1.0.0', title: 'APM' }, + }) + ).rejects.toThrowError('Output type "kafka" is not usable with package "apm"'); + }); + + it('should not throw if valid license and valid output_id is provided', async () => { + jest.spyOn(licenseService, 'hasAtLeast').mockReturnValue(true); + jest + .spyOn(outputService, 'get') + .mockResolvedValueOnce({ id: 'es-output', type: 'elasticsearch' } as any); + await expect( + preflightCheckPackagePolicy(soClient, { + ...testPolicy, + output_id: 'es-output', + }) + ).resolves.not.toThrow(); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/package_policies/utils.ts b/x-pack/plugins/fleet/server/services/package_policies/utils.ts index cf2d69c8874b96..61ff380fec559f 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/utils.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/utils.ts @@ -6,8 +6,18 @@ */ import type { SavedObject } from '@kbn/core-saved-objects-common/src/server_types'; +import type { SavedObjectsClientContract } from '@kbn/core/server'; -import type { PackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { + LICENCE_FOR_OUTPUT_PER_INTEGRATION, + LICENCE_FOR_MULTIPLE_AGENT_POLICIES, +} from '../../../common/constants'; +import { getAllowedOutputTypesForIntegration } from '../../../common/services/output_helpers'; +import type { PackagePolicy, NewPackagePolicy, PackagePolicySOAttributes } from '../../types'; +import { PackagePolicyMultipleAgentPoliciesError, PackagePolicyOutputError } from '../../errors'; +import { licenseService } from '../license'; +import { outputService } from '../output'; +import { appContextService } from '../app_context'; export const mapPackagePolicySavedObjectToPackagePolicy = ({ /* eslint-disable @typescript-eslint/naming-convention */ @@ -21,6 +31,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, // `package` is a reserved keyword package: packageInfo, inputs, @@ -45,6 +56,7 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ is_managed, policy_id, policy_ids, + output_id, package: packageInfo, inputs, vars, @@ -59,3 +71,69 @@ export const mapPackagePolicySavedObjectToPackagePolicy = ({ created_by, }; }; + +export async function preflightCheckPackagePolicy( + soClient: SavedObjectsClientContract, + packagePolicy: PackagePolicy | NewPackagePolicy +) { + // If package policy has multiple agent policies, check if they can be used + const { canUseReusablePolicies, errorMessage: canUseMultipleAgentPoliciesErrorMessage } = + canUseMultipleAgentPolicies(); + if ((packagePolicy.policy_ids ?? []).length > 1 && !canUseReusablePolicies) { + throw new PackagePolicyMultipleAgentPoliciesError(canUseMultipleAgentPoliciesErrorMessage); + } + + // If package policy has an output_id, check if it can be used + if (packagePolicy.output_id && packagePolicy.package) { + const { canUseOutputForIntegrationResult, errorMessage: outputForIntegrationErrorMessage } = + await canUseOutputForIntegration( + soClient, + packagePolicy.package.name, + packagePolicy.output_id + ); + if (!canUseOutputForIntegrationResult && outputForIntegrationErrorMessage) { + throw new PackagePolicyOutputError(outputForIntegrationErrorMessage); + } + } +} + +export function canUseMultipleAgentPolicies() { + const hasEnterpriseLicence = licenseService.hasAtLeast(LICENCE_FOR_MULTIPLE_AGENT_POLICIES); + const { enableReusableIntegrationPolicies } = appContextService.getExperimentalFeatures(); + + return { + canUseReusablePolicies: hasEnterpriseLicence && enableReusableIntegrationPolicies, + errorMessage: !hasEnterpriseLicence + ? 'Reusable integration policies are only available with an Enterprise license' + : 'Reusable integration policies are not supported', + }; +} + +export async function canUseOutputForIntegration( + soClient: SavedObjectsClientContract, + packageName: string, + outputId: string +) { + const hasAllowedLicense = licenseService.hasAtLeast(LICENCE_FOR_OUTPUT_PER_INTEGRATION); + if (!hasAllowedLicense) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output per integration is only available with an ${LICENCE_FOR_OUTPUT_PER_INTEGRATION} license`, + }; + } + + const allowedOutputTypes = getAllowedOutputTypesForIntegration(packageName); + const output = await outputService.get(soClient, outputId); + + if (!allowedOutputTypes.includes(output.type)) { + return { + canUseOutputForIntegrationResult: false, + errorMessage: `Output type "${output.type}" is not usable with package "${packageName}"`, + }; + } + + return { + canUseOutputForIntegrationResult: true, + errorMessage: null, + }; +} diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index a62a5048ec567d..9505edd3556cb3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -2921,6 +2921,7 @@ describe('Package policy service', () => { version: 'WzYyMzcsMV0=', name: 'my-cis_kubernetes_benchmark', namespace: 'default', + output_id: null, description: '', package: { name: 'cis_kubernetes_benchmark', @@ -5096,6 +5097,61 @@ describe('Package policy service', () => { ); }); }); + + describe('removeOutputFromAll', () => { + it('should update policies using deleted output', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const updateSpy = jest.spyOn(packagePolicyService, 'update'); + soClient.find.mockResolvedValue({ + saved_objects: [ + { + id: 'package-policy-1', + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }, + ], + } as any); + soClient.get.mockImplementation((type, id, options): any => { + if (id === 'package-policy-1') { + return Promise.resolve({ + id, + attributes: { + name: 'policy1', + enabled: true, + policy_ids: ['agent-policy-1'], + output_id: 'output-id-123', + inputs: [], + package: { name: 'test-package', version: '1.0.0' }, + }, + }); + } + }); + + await packagePolicyService.removeOutputFromAll(soClient, esClient, 'output-id-123'); + + expect(updateSpy).toHaveBeenCalledTimes(1); + expect(updateSpy).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 'package-policy-1', + { + name: 'policy1', + enabled: true, + policy_id: 'agent-policy-1', + policy_ids: ['agent-policy-1'], + output_id: null, + inputs: [], + } + ); + }); + }); }); describe('getUpgradeDryRunDiff', () => { diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index d9605880c1572e..65495d51f0f60c 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -115,6 +115,7 @@ import { sendTelemetryEvents } from './upgrade_sender'; import { handleExperimentalDatastreamFeatureOptIn, mapPackagePolicySavedObjectToPackagePolicy, + preflightCheckPackagePolicy, } from './package_policies'; import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { @@ -131,7 +132,7 @@ import { isSecretStorageEnabled, } from './secrets'; import { getPackageAssetsMap } from './epm/packages/get'; -import { validateOutputForNewPackagePolicy } from './agent_policies/outputs_helpers'; +import { validateAgentPolicyOutputForIntegration } from './agent_policies/outputs_helpers'; import type { PackagePolicyClientFetchAllItemIdsOptions } from './package_policy_service'; import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; @@ -212,6 +213,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { logger.debug(`Creating new package policy`); this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); let enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyCreate', @@ -228,8 +230,9 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const agentPolicy = await agentPolicyService.get(soClient, policyId, true); agentPolicies.push(agentPolicy); - if (agentPolicy && enrichedPackagePolicy.package?.name) { - await validateOutputForNewPackagePolicy( + // If package policy did not set an output_id, see if the agent policy's output is compatible + if (!packagePolicy.output_id && agentPolicy && enrichedPackagePolicy.package?.name) { + await validateAgentPolicyOutputForIntegration( soClient, agentPolicy, enrichedPackagePolicy.package?.name @@ -408,14 +411,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { if (!packagePolicy.id) { packagePolicy.id = SavedObjectsUtils.generateId(); } - - this.keepPolicyIdInSync(packagePolicy); - auditLoggingService.writeCustomSoAuditLog({ action: 'create', id: packagePolicy.id, savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, }); + + this.keepPolicyIdInSync(packagePolicy); + await preflightCheckPackagePolicy(soClient, packagePolicy); } const agentPolicyIds = new Set(packagePolicies.flatMap((pkgPolicy) => pkgPolicy.policy_ids)); @@ -471,6 +474,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const { pkgInfo, assetsMap } = packageInfoAndAsset; + validatePackagePolicyOrThrow(packagePolicy, pkgInfo); inputs = pkgInfo ? await _compilePackagePolicyInputs( @@ -822,6 +826,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { try { logger.debug(`Starting update of package policy ${id}`); + await preflightCheckPackagePolicy(soClient, packagePolicyUpdate); enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( 'packagePolicyUpdate', packagePolicyUpdate, @@ -1052,6 +1057,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { const id = packagePolicyUpdate.id; this.keepPolicyIdInSync(packagePolicyUpdate); const packagePolicy = { ...packagePolicyUpdate, name: packagePolicyUpdate.name.trim() }; + await preflightCheckPackagePolicy(soClient, packagePolicy); const oldPackagePolicy = oldPackagePolicies.find((p) => p.id === id); if (!oldPackagePolicy) { throw new PackagePolicyNotFoundError('Package policy not found'); @@ -1762,6 +1768,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }, policy_id: newPolicy.policy_id ?? agentPolicyId, policy_ids: newPolicy.policy_ids ?? [agentPolicyId], + output_id: newPolicy.output_id, inputs: newPolicy.inputs[0]?.streams ? newPolicy.inputs : inputs, vars: newPolicy.vars || newPP.vars, }; @@ -1982,6 +1989,76 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } } + public async removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ) { + const packagePolicies = ( + await soClient.find({ + type: SAVED_OBJECT_TYPE, + fields: ['name', 'enabled', 'policy_ids', 'inputs', 'output_id'], + searchFields: ['output_id'], + search: escapeSearchQueryPhrase(outputId), + perPage: SO_SEARCH_LIMIT, + }) + ).saved_objects.map(mapPackagePolicySavedObjectToPackagePolicy); + + if (packagePolicies.length > 0) { + const getPackagePolicyUpdate = (packagePolicy: PackagePolicy) => ({ + name: packagePolicy.name, + enabled: packagePolicy.enabled, + policy_ids: packagePolicy.policy_ids, + inputs: packagePolicy.inputs, + output_id: packagePolicy.output_id === outputId ? null : packagePolicy.output_id, + }); + + // Validate that the new cleared/default output is valid for the package policies + // (from each of the associated agent policies) before updating any of them + await pMap( + packagePolicies, + async (packagePolicy) => { + const existingPackagePolicy = await this.get(soClient, packagePolicy.id); + + if (!existingPackagePolicy) { + throw new PackagePolicyNotFoundError('Package policy not found'); + } + + for (const policyId of packagePolicy.policy_ids) { + if (packagePolicy.package?.name) { + const agentPolicy = await agentPolicyService.get(soClient, policyId, true); + if (agentPolicy) { + await validateAgentPolicyOutputForIntegration( + soClient, + agentPolicy, + packagePolicy.package.name, + false + ); + } + } + } + }, + { + concurrency: 50, + } + ); + await pMap( + packagePolicies, + (packagePolicy) => { + return this.update( + soClient, + esClient, + packagePolicy.id, + getPackagePolicyUpdate(packagePolicy) + ); + }, + { + concurrency: 50, + } + ); + } + } + fetchAllItemIds( soClient: SavedObjectsClientContract, { perPage = 1000, kuery }: PackagePolicyClientFetchAllItemIdsOptions = {} diff --git a/x-pack/plugins/fleet/server/services/package_policy_service.ts b/x-pack/plugins/fleet/server/services/package_policy_service.ts index 1462d451388b52..fed46872ab6cba 100644 --- a/x-pack/plugins/fleet/server/services/package_policy_service.ts +++ b/x-pack/plugins/fleet/server/services/package_policy_service.ts @@ -219,6 +219,18 @@ export interface PackagePolicyClient { experimentalDataStreamFeatures: ExperimentalDataStreamFeature[]; }>; + /** + * Remove an output from all package policies that are using it, and replace the output by the default ones. + * @param soClient + * @param esClient + * @param outputId + */ + removeOutputFromAll( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + outputId: string + ): Promise; + /** * Returns an `AsyncIterable` for retrieving all integration policy IDs * @param soClient diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts index f6feffd24df535..3088814c8f8a32 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/outputs.test.ts @@ -6,6 +6,7 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + import { appContextService } from '..'; import type { PreconfiguredOutput } from '../../../common/types'; @@ -31,6 +32,9 @@ const mockedOutputService = outputService as jest.Mocked; jest.mock('../app_context', () => ({ appContextService: { getInternalUserSOClientWithoutSpaceExtension: jest.fn(), + getExperimentalFeatures: () => ({ + useSpaceAwareness: true, + }), getLogger: () => new Proxy( {}, @@ -60,6 +64,9 @@ describe('output preconfiguration', () => { per_page: 0, total: 0, }); + internalSoClientWithoutSpaceExtension.bulkGet.mockResolvedValue({ + saved_objects: [], + }); mockedOutputService.create.mockReset(); mockedOutputService.update.mockReset(); mockedOutputService.delete.mockReset(); diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts index 54db36b49d0430..c80cb45c84ff9a 100644 --- a/x-pack/plugins/fleet/server/types/models/package_policy.ts +++ b/x-pack/plugins/fleet/server/types/models/package_policy.ts @@ -98,6 +98,7 @@ const PackagePolicyBaseSchema = { namespace: schema.maybe(PackagePolicyNamespaceSchema), policy_id: schema.maybe(schema.string()), policy_ids: schema.maybe(schema.arrayOf(schema.string())), + output_id: schema.nullable(schema.maybe(schema.string())), enabled: schema.boolean(), is_managed: schema.maybe(schema.boolean()), package: schema.maybe( @@ -109,8 +110,6 @@ const PackagePolicyBaseSchema = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf(schema.object(PackagePolicyInputsSchema)), vars: schema.maybe(ConfigRecordSchema), overrides: schema.maybe( @@ -152,8 +151,6 @@ const CreatePackagePolicyProps = { requires_root: schema.maybe(schema.boolean()), }) ), - // Deprecated TODO create remove issue - output_id: schema.maybe(schema.string()), inputs: schema.arrayOf( schema.object({ ...PackagePolicyInputsSchema, @@ -191,6 +188,7 @@ export const SimplifiedPackagePolicyBaseSchema = schema.object({ name: schema.string(), description: schema.maybe(schema.string()), namespace: schema.maybe(schema.string()), + output_id: schema.nullable(schema.maybe(schema.string())), vars: schema.maybe(SimplifiedVarsSchema), inputs: schema.maybe( schema.recordOf( diff --git a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts index 6216eb4bbd3269..79586b1885ed97 100644 --- a/x-pack/plugins/fleet/server/types/models/preconfiguration.ts +++ b/x-pack/plugins/fleet/server/types/models/preconfiguration.ts @@ -162,6 +162,7 @@ export const PreconfiguredAgentPoliciesSchema = schema.arrayOf( }), description: schema.maybe(schema.string()), namespace: schema.maybe(PackagePolicyNamespaceSchema), + output_id: schema.nullable(schema.maybe(schema.string())), inputs: schema.maybe( schema.arrayOf( schema.object({ diff --git a/x-pack/plugins/fleet/server/types/so_attributes.ts b/x-pack/plugins/fleet/server/types/so_attributes.ts index 79640a0f90e129..ef39759b206f55 100644 --- a/x-pack/plugins/fleet/server/types/so_attributes.ts +++ b/x-pack/plugins/fleet/server/types/so_attributes.ts @@ -121,6 +121,8 @@ export interface PackagePolicySOAttributes { inputs: PackagePolicyInput[]; policy_id?: string; policy_ids: string[]; + // Nullable to allow user to reset to default outputs + output_id?: string | null; updated_at: string; updated_by: string; description?: string; diff --git a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap index 9112ad20ad8609..a4d255613133ef 100644 --- a/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap +++ b/x-pack/test/fleet_api_integration/apis/agent_policy/__snapshots__/agent_policy.snap @@ -30,6 +30,7 @@ Object { "enabled": true, "name": "system-1", "namespace": "default", + "output_id": null, "package": Object { "name": "system", "requires_root": true, diff --git a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts index 76585c1f0099ad..b860a774ba1220 100644 --- a/x-pack/test/fleet_api_integration/apis/outputs/crud.ts +++ b/x-pack/test/fleet_api_integration/apis/outputs/crud.ts @@ -120,7 +120,10 @@ export default function (providerContext: FtrProviderContext) { } }; - const createAgentPolicy = async (spaceId?: string): Promise => { + const createAgentPolicy = async ( + spaceId?: string, + dataOutputId?: string + ): Promise => { const { body: testPolicyRes } = await supertest .post(spaceId ? `/s/${spaceId}/api/fleet/agent_policies` : `/api/fleet/agent_policies`) .set('kbn-xsrf', 'xxxx') @@ -128,6 +131,7 @@ export default function (providerContext: FtrProviderContext) { name: `test ${uuidV4()}`, description: '', namespace: 'default', + ...(dataOutputId ? { data_output_id: dataOutputId } : {}), }) .expect(200); @@ -148,6 +152,33 @@ export default function (providerContext: FtrProviderContext) { .expect(200); }; + const createPackagePolicy = async ( + agentPolicyIds: string[], + spaceId?: string, + outputId?: string + ): Promise => { + const { body: testPolicyRes } = await supertest + .post(spaceId ? `/s/${spaceId}/api/fleet/package_policies` : `/api/fleet/package_policies`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: `package ${uuidV4()}`, + description: '', + namespace: 'default', + policy_ids: agentPolicyIds, + enabled: true, + inputs: [], + package: { + name: 'filetest', + title: 'For File Tests', + version: '0.1.0', + }, + ...(outputId ? { output_id: outputId } : {}), + }) + .expect(200); + + return testPolicyRes; + }; + const getAgentPolicy = async ( policyId: string, spaceId?: string @@ -396,7 +427,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Logstash output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -417,7 +448,7 @@ export default function (providerContext: FtrProviderContext) { }) .expect(400); expect(body.message).to.eql( - 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new ElasticSearch output.' + 'Kafka output cannot be used with Fleet Server integration in Fleet Server policy 1. Please create a new Elasticsearch output.' ); }); @@ -765,10 +796,28 @@ export default function (providerContext: FtrProviderContext) { }); it('should bump all policies in all spaces if updating the default output', async () => { - const [policy1, policy2, policy3] = await Promise.all([ - createAgentPolicy(), + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies with default output under agent policies not using default output + // to ensure that those agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy2.item.id], undefined, defaultOutputId), + createPackagePolicy([policy4.item.id], TEST_SPACE_ID, defaultOutputId), ]); await supertest @@ -781,20 +830,79 @@ export default function (providerContext: FtrProviderContext) { }) .expect(200); - const [updatedPolicy1, updatedPolicy2, updatedPolicy3] = await Promise.all([ + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ getAgentPolicy(policy1.item.id), getAgentPolicy(policy2.item.id), getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 1); - expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 2); expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 1); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 2); + + // cleanup + await Promise.all([ + deleteAgentPolicy(policy1.item.id), + deleteAgentPolicy(policy2.item.id), + deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + }); + + it('should bump all policies in all spaces if updating non-default output', async () => { + const { body: nonDefaultOutput } = await supertest + .post(`/api/fleet/outputs`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Nondefault Output', + type: 'elasticsearch', + hosts: ['https://test.fr'], + }) + .expect(200); + + const [policy1, policy2, policy3, policy4] = await Promise.all([ + createAgentPolicy(), + createAgentPolicy(undefined, nonDefaultOutput.item.id), + createAgentPolicy(TEST_SPACE_ID), + createAgentPolicy(TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + // Create package policies under agent policies using default output to ensure those + // agent policies still get bumped + await Promise.all([ + createPackagePolicy([policy1.item.id], undefined, nonDefaultOutput.item.id), + createPackagePolicy([policy3.item.id], TEST_SPACE_ID, nonDefaultOutput.item.id), + ]); + + await supertest + .put(`/api/fleet/outputs/${nonDefaultOutput.item.id}`) + .set('kbn-xsrf', 'xxxx') + .send({ + name: 'Updated Nondefault Output', + type: 'elasticsearch', + hosts: ['http://test.fr:443'], + }) + .expect(200); + + const [updatedPolicy1, updatedPolicy2, updatedPolicy3, updatedPolicy4] = await Promise.all([ + getAgentPolicy(policy1.item.id), + getAgentPolicy(policy2.item.id), + getAgentPolicy(policy3.item.id, TEST_SPACE_ID), + getAgentPolicy(policy4.item.id, TEST_SPACE_ID), + ]); + + expect(updatedPolicy1.item.revision).to.eql(policy1.item.revision + 2); + expect(updatedPolicy2.item.revision).to.eql(policy2.item.revision + 1); + expect(updatedPolicy3.item.revision).to.eql(policy3.item.revision + 2); + expect(updatedPolicy4.item.revision).to.eql(policy4.item.revision + 1); // cleanup await Promise.all([ deleteAgentPolicy(policy1.item.id), deleteAgentPolicy(policy2.item.id), deleteAgentPolicy(policy3.item.id, TEST_SPACE_ID), + deleteAgentPolicy(policy4.item.id, TEST_SPACE_ID), ]); }); });