diff --git a/docs/user/alerting/action-types/email.asciidoc b/docs/user/alerting/action-types/email.asciidoc index 83e7edc5a016a6..d7a9373a6e2a99 100644 --- a/docs/user/alerting/action-types/email.asciidoc +++ b/docs/user/alerting/action-types/email.asciidoc @@ -14,7 +14,7 @@ Name:: The name of the connector. The name is used to identify a connector Sender:: The from address for all emails sent with this connector, specified in `user@host-name` format. Host:: Host name of the service provider. If you are using the <> setting, make sure this hostname is added to the allowed hosts. Port:: The port to connect to on the service provider. -Secure:: If true the connection will use TLS when connecting to the service provider. See https://nodemailer.com/smtp/#tls-options[nodemailer TLS documentation] for more information. +Secure:: If true, the connection will use TLS when connecting to the service provider. Refer to the https://nodemailer.com/smtp/#tls-options[Nodemailer TLS documentation] for more information. If not true, the connection will initially connect over TCP, then attempt to switch to TLS via the SMTP STARTTLS command. Username:: username for 'login' type authentication. Password:: password for 'login' type authentication. @@ -92,6 +92,8 @@ systems, refer to: * <> * <> +For other email servers, you can check the list of well-known services that Nodemailer supports in the JSON file https://github.com/nodemailer/nodemailer/blob/master/lib/well-known/services.json[well-known/services.json]. The properties of the objects in those files — `host`, `port`, and `secure` — correspond to the same email action configuration properties. A missing `secure` property in the "well-known/services.json" file is considered `false`. Typically, `port: 465` uses `secure: true`, and `port: 25` and `port: 587` use `secure: false`. + [float] [[gmail]] ===== Sending email from Gmail @@ -109,7 +111,6 @@ https://mail.google.com[Gmail] SMTP service: user: password: -------------------------------------------------- -// CONSOLE If you get an authentication error that indicates that you need to continue the sign-in process from a web browser when the action attempts to send email, you need @@ -131,9 +132,9 @@ https://www.outlook.com/[Outlook.com] SMTP service: [source,text] -------------------------------------------------- config: - host: smtp-mail.outlook.com - port: 465 - secure: true + host: smtp.office365.com + port: 587 + secure: false secrets: user: password: @@ -163,7 +164,7 @@ secrets: user: password: -------------------------------------------------- -<1> `smtp.host` varies depending on the region +<1> `config.host` varies depending on the region NOTE: You must use your Amazon SES SMTP credentials to send email through Amazon SES. For more information, see diff --git a/x-pack/plugins/fleet/common/types/models/agent.ts b/x-pack/plugins/fleet/common/types/models/agent.ts index 57c42d887bc83c..2e18d427272ce5 100644 --- a/x-pack/plugins/fleet/common/types/models/agent.ts +++ b/x-pack/plugins/fleet/common/types/models/agent.ts @@ -130,8 +130,8 @@ interface AgentBase { enrolled_at: string; unenrolled_at?: string; unenrollment_started_at?: string; - upgraded_at?: string; - upgrade_started_at?: string; + upgraded_at?: string | null; + upgrade_started_at?: string | null; access_api_key_id?: string; default_api_key?: string; default_api_key_id?: string; @@ -155,3 +155,163 @@ export interface AgentSOAttributes extends AgentBase { current_error_events?: string; packages?: string[]; } + +// Generated from FleetServer schema.json + +/** + * An Elastic Agent that has enrolled into Fleet + */ +export interface FleetServerAgent { + /** + * The version of the document in the index + */ + _version?: number; + /** + * Shared ID + */ + shared_id?: string; + /** + * Type + */ + type: AgentType; + /** + * Active flag + */ + active: boolean; + /** + * Date/time the Elastic Agent enrolled + */ + enrolled_at: string; + /** + * Date/time the Elastic Agent unenrolled + */ + unenrolled_at?: string; + /** + * Date/time the Elastic Agent unenrolled started + */ + unenrollment_started_at?: string; + /** + * Date/time the Elastic Agent was last upgraded + */ + upgraded_at?: string | null; + /** + * Date/time the Elastic Agent started the current upgrade + */ + upgrade_started_at?: string | null; + /** + * ID of the API key the Elastic Agent must used to contact Fleet Server + */ + access_api_key_id?: string; + agent?: FleetServerAgentMetadata; + /** + * User provided metadata information for the Elastic Agent + */ + user_provided_metadata: AgentMetadata; + /** + * Local metadata information for the Elastic Agent + */ + local_metadata: AgentMetadata; + /** + * The policy ID for the Elastic Agent + */ + policy_id?: string; + /** + * The current policy revision_idx for the Elastic Agent + */ + policy_revision_idx?: number | null; + /** + * The current policy coordinator for the Elastic Agent + */ + policy_coordinator_idx?: number; + /** + * Date/time the Elastic Agent was last updated + */ + last_updated?: string; + /** + * Date/time the Elastic Agent checked in last time + */ + last_checkin?: string; + /** + * Lst checkin status + */ + last_checkin_status?: 'error' | 'online' | 'degraded' | 'updating'; + /** + * ID of the API key the Elastic Agent uses to authenticate with elasticsearch + */ + default_api_key_id?: string; + /** + * API key the Elastic Agent uses to authenticate with elasticsearch + */ + default_api_key?: string; + /** + * Date/time the Elastic Agent was last updated + */ + updated_at?: string; + /** + * Packages array + */ + packages?: string[]; + /** + * The last acknowledged action sequence number for the Elastic Agent + */ + action_seq_no?: number; +} +/** + * An Elastic Agent metadata + */ +export interface FleetServerAgentMetadata { + /** + * The unique identifier for the Elastic Agent + */ + id: string; + /** + * The version of the Elastic Agent + */ + version: string; + [k: string]: any; +} + +/** + * An Elastic Agent action + */ +export interface FleetServerAgentAction { + /** + * The unique identifier for action document + */ + _id?: string; + /** + * The action sequence number + */ + _seq_no?: number; + /** + * The unique identifier for the Elastic Agent action. There could be multiple documents with the same action_id if the action is split into two separate documents. + */ + action_id?: string; + /** + * Date/time the action was created + */ + '@timestamp'?: string; + /** + * The action expiration date/time + */ + expiration?: string; + /** + * The action type. APP_ACTION is the value for the actions that suppose to be routed to the endpoints/beats. + */ + type?: string; + /** + * The input identifier the actions should be routed to. + */ + input_id?: string; + /** + * The Agent IDs the action is intended for. No support for json.RawMessage with the current generator. Could be useful to lazy parse the agent ids + */ + agents?: string[]; + /** + * The opaque payload. + */ + data?: { + [k: string]: unknown; + }; + [k: string]: unknown; +} diff --git a/x-pack/plugins/fleet/server/routes/agent/acks_handlers.ts b/x-pack/plugins/fleet/server/routes/agent/acks_handlers.ts index 2d7c884edad83c..22b5035378a20d 100644 --- a/x-pack/plugins/fleet/server/routes/agent/acks_handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent/acks_handlers.ts @@ -20,7 +20,7 @@ export const postAgentAcksHandlerBuilder = function ( try { const soClient = ackService.getSavedObjectsClientContract(request); const esClient = ackService.getElasticsearchClientContract(); - const agent = await ackService.authenticateAgentWithAccessToken(soClient, request); + const agent = await ackService.authenticateAgentWithAccessToken(soClient, esClient, request); const agentEvents = request.body.events as AgentEvent[]; // validate that all events are for the authorized agent obtained from the api key diff --git a/x-pack/plugins/fleet/server/routes/agent/actions_handlers.ts b/x-pack/plugins/fleet/server/routes/agent/actions_handlers.ts index bf0cfd2d476dd8..d032945245faf0 100644 --- a/x-pack/plugins/fleet/server/routes/agent/actions_handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent/actions_handlers.ts @@ -30,7 +30,7 @@ export const postNewAgentActionHandlerBuilder = function ( const newAgentAction = request.body.action; - const savedAgentAction = await actionsService.createAgentAction(soClient, { + const savedAgentAction = await actionsService.createAgentAction(soClient, esClient, { created_at: new Date().toISOString(), ...newAgentAction, agent_id: agent.id, diff --git a/x-pack/plugins/fleet/server/routes/agent/handlers.ts b/x-pack/plugins/fleet/server/routes/agent/handlers.ts index 411da6da0223c6..cd91e8c325c066 100644 --- a/x-pack/plugins/fleet/server/routes/agent/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent/handlers.ts @@ -132,8 +132,8 @@ export const updateAgentHandler: RequestHandler< const esClient = context.core.elasticsearch.client.asCurrentUser; try { - await AgentService.updateAgent(soClient, request.params.agentId, { - userProvidedMetatada: request.body.user_provided_metadata, + await AgentService.updateAgent(soClient, esClient, request.params.agentId, { + user_provided_metadata: request.body.user_provided_metadata, }); const agent = await AgentService.getAgent(soClient, esClient, request.params.agentId); @@ -164,12 +164,13 @@ export const postAgentCheckinHandler: RequestHandler< try { const soClient = appContextService.getInternalUserSOClient(request); const esClient = appContextService.getInternalUserESClient(); - const agent = await AgentService.authenticateAgentWithAccessToken(soClient, request); + const agent = await AgentService.authenticateAgentWithAccessToken(soClient, esClient, request); const abortController = new AbortController(); request.events.aborted$.subscribe(() => { abortController.abort(); }); const signal = abortController.signal; + const { actions } = await AgentService.agentCheckin( soClient, esClient, @@ -205,8 +206,13 @@ export const postAgentEnrollHandler: RequestHandler< > = async (context, request, response) => { try { const soClient = appContextService.getInternalUserSOClient(request); + const esClient = context.core.elasticsearch.client.asInternalUser; const { apiKeyId } = APIKeyService.parseApiKeyFromHeaders(request.headers); - const enrollmentAPIKey = await APIKeyService.getEnrollmentAPIKeyById(soClient, apiKeyId); + const enrollmentAPIKey = await APIKeyService.getEnrollmentAPIKeyById( + soClient, + esClient, + apiKeyId + ); if (!enrollmentAPIKey || !enrollmentAPIKey.active) { return response.unauthorized({ @@ -311,21 +317,16 @@ export const postBulkAgentsReassignHandler: RequestHandler< const soClient = context.core.savedObjects.client; const esClient = context.core.elasticsearch.client.asInternalUser; try { - // Reassign by array of IDs - const result = Array.isArray(request.body.agents) - ? await AgentService.reassignAgents( - soClient, - esClient, - { agentIds: request.body.agents }, - request.body.policy_id - ) - : await AgentService.reassignAgents( - soClient, - esClient, - { kuery: request.body.agents }, - request.body.policy_id - ); - const body: PostBulkAgentReassignResponse = result.saved_objects.reduce((acc, so) => { + const results = await AgentService.reassignAgents( + soClient, + esClient, + Array.isArray(request.body.agents) + ? { agentIds: request.body.agents } + : { kuery: request.body.agents }, + request.body.policy_id + ); + + const body: PostBulkAgentReassignResponse = results.items.reduce((acc, so) => { return { ...acc, [so.id]: { diff --git a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts index 0215b8f27b3932..086a9411f20b8d 100644 --- a/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts +++ b/x-pack/plugins/fleet/server/routes/agent/upgrade_handler.ts @@ -8,18 +8,13 @@ import { RequestHandler } from 'src/core/server'; import { TypeOf } from '@kbn/config-schema'; import semverCoerce from 'semver/functions/coerce'; -import { - AgentSOAttributes, - PostAgentUpgradeResponse, - PostBulkAgentUpgradeResponse, -} from '../../../common/types'; +import { PostAgentUpgradeResponse, PostBulkAgentUpgradeResponse } from '../../../common/types'; import { PostAgentUpgradeRequestSchema, PostBulkAgentUpgradeRequestSchema } from '../../types'; import * as AgentService from '../../services/agents'; import { appContextService } from '../../services'; import { defaultIngestErrorHandler } from '../../errors'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; -import { savedObjectToAgent } from '../../services/agents/saved_objects'; import { isAgentUpgradeable } from '../../../common/services'; +import { getAgent } from '../../services/agents'; export const postAgentUpgradeHandler: RequestHandler< TypeOf, @@ -27,6 +22,7 @@ export const postAgentUpgradeHandler: RequestHandler< TypeOf > = async (context, request, response) => { const soClient = context.core.savedObjects.client; + const esClient = context.core.elasticsearch.client.asInternalUser; const { version, source_uri: sourceUri, force } = request.body; const kibanaVersion = appContextService.getKibanaVersion(); try { @@ -39,12 +35,8 @@ export const postAgentUpgradeHandler: RequestHandler< }, }); } - - const agentSO = await soClient.get( - AGENT_SAVED_OBJECT_TYPE, - request.params.agentId - ); - if (agentSO.attributes.unenrollment_started_at || agentSO.attributes.unenrolled_at) { + const agent = await getAgent(soClient, esClient, request.params.agentId); + if (agent.unenrollment_started_at || agent.unenrolled_at) { return response.customError({ statusCode: 400, body: { @@ -53,7 +45,6 @@ export const postAgentUpgradeHandler: RequestHandler< }); } - const agent = savedObjectToAgent(agentSO); if (!force && !isAgentUpgradeable(agent, kibanaVersion)) { return response.customError({ statusCode: 400, @@ -66,6 +57,7 @@ export const postAgentUpgradeHandler: RequestHandler< try { await AgentService.sendUpgradeAgentAction({ soClient, + esClient, agentId: request.params.agentId, version, sourceUri, diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index dfe5c19bc417b5..ca131efeff68cc 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -488,17 +488,17 @@ class AgentPolicyService { soClient: SavedObjectsClientContract, agentPolicyId: string ) { - return appContextService.getConfig()?.agents.fleetServerEnabled - ? this.createFleetPolicyChangeFleetServer( - soClient, - appContextService.getInternalUserESClient(), - agentPolicyId - ) - : this.createFleetPolicyChangeActionSO(soClient, agentPolicyId); + const esClient = appContextService.getInternalUserESClient(); + if (appContextService.getConfig()?.agents?.fleetServerEnabled) { + await this.createFleetPolicyChangeFleetServer(soClient, esClient, agentPolicyId); + } + + return this.createFleetPolicyChangeActionSO(soClient, esClient, agentPolicyId); } public async createFleetPolicyChangeActionSO( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agentPolicyId: string ) { // If Agents is not setup skip the creation of POLICY_CHANGE agent actions @@ -518,7 +518,7 @@ class AgentPolicyService { return acc; }, []); - await createAgentPolicyAction(soClient, { + await createAgentPolicyAction(soClient, esClient, { type: 'POLICY_CHANGE', data: { policy }, ack_data: { packages }, diff --git a/x-pack/plugins/fleet/server/services/agents/acks.test.ts b/x-pack/plugins/fleet/server/services/agents/acks.test.ts index c1a6067195c979..5aec696f5e1440 100644 --- a/x-pack/plugins/fleet/server/services/agents/acks.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/acks.test.ts @@ -106,19 +106,19 @@ describe('test agent acks services', () => { } as AgentEvent, ] ); - expect(mockSavedObjectsClient.bulkUpdate).toBeCalled(); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0]).toHaveLength(1); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0][0]).toMatchInlineSnapshot(` - Object { - "attributes": Object { + expect(mockSavedObjectsClient.bulkUpdate).not.toBeCalled(); + expect(mockSavedObjectsClient.update).toBeCalled(); + expect(mockSavedObjectsClient.update.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "fleet-agents", + "id", + Object { "packages": Array [ "system", ], "policy_revision": 4, }, - "id": "id", - "type": "fleet-agents", - } + ] `); }); @@ -168,19 +168,19 @@ describe('test agent acks services', () => { } as AgentEvent, ] ); - expect(mockSavedObjectsClient.bulkUpdate).toBeCalled(); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0]).toHaveLength(1); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0][0]).toMatchInlineSnapshot(` - Object { - "attributes": Object { + expect(mockSavedObjectsClient.bulkUpdate).not.toBeCalled(); + expect(mockSavedObjectsClient.update).toBeCalled(); + expect(mockSavedObjectsClient.update.mock.calls[0]).toMatchInlineSnapshot(` + Array [ + "fleet-agents", + "id", + Object { "packages": Array [ "system", ], "policy_revision": 4, }, - "id": "id", - "type": "fleet-agents", - } + ] `); }); @@ -230,8 +230,8 @@ describe('test agent acks services', () => { } as AgentEvent, ] ); - expect(mockSavedObjectsClient.bulkUpdate).toBeCalled(); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0]).toHaveLength(0); + expect(mockSavedObjectsClient.bulkUpdate).not.toBeCalled(); + expect(mockSavedObjectsClient.update).not.toBeCalled(); }); it('should not update config field on the agent if a policy change for an old revision is acknowledged', async () => { @@ -277,8 +277,8 @@ describe('test agent acks services', () => { } as AgentEvent, ] ); - expect(mockSavedObjectsClient.bulkUpdate).toBeCalled(); - expect(mockSavedObjectsClient.bulkUpdate.mock.calls[0][0]).toHaveLength(0); + expect(mockSavedObjectsClient.bulkUpdate).not.toBeCalled(); + expect(mockSavedObjectsClient.update).not.toBeCalled(); }); it('should fail for actions that cannot be found on agent actions list', async () => { diff --git a/x-pack/plugins/fleet/server/services/agents/acks.ts b/x-pack/plugins/fleet/server/services/agents/acks.ts index a09107b90a0156..c639a9b0332ac6 100644 --- a/x-pack/plugins/fleet/server/services/agents/acks.ts +++ b/x-pack/plugins/fleet/server/services/agents/acks.ts @@ -24,14 +24,11 @@ import { AgentSOAttributes, AgentActionSOAttributes, } from '../../types'; -import { - AGENT_EVENT_SAVED_OBJECT_TYPE, - AGENT_SAVED_OBJECT_TYPE, - AGENT_ACTION_SAVED_OBJECT_TYPE, -} from '../../constants'; +import { AGENT_EVENT_SAVED_OBJECT_TYPE, AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../constants'; import { getAgentActionByIds } from './actions'; import { forceUnenrollAgent } from './unenroll'; import { ackAgentUpgraded } from './upgrade'; +import { updateAgent } from './crud'; const ALLOWED_ACKNOWLEDGEMENT_TYPE: string[] = ['ACTION_RESULT']; @@ -87,26 +84,23 @@ export async function acknowledgeAgentActions( const upgradeAction = actions.find((action) => action.type === 'UPGRADE'); if (upgradeAction) { - await ackAgentUpgraded(soClient, upgradeAction); + await ackAgentUpgraded(soClient, esClient, upgradeAction); } const configChangeAction = getLatestConfigChangePolicyActionIfUpdated(agent, actions); - await soClient.bulkUpdate([ - ...(configChangeAction - ? [ - { - type: AGENT_SAVED_OBJECT_TYPE, - id: agent.id, - attributes: { - policy_revision: configChangeAction.policy_revision, - packages: configChangeAction?.ack_data?.packages, - }, - }, - ] - : []), - ...buildUpdateAgentActionSentAt(agentActionsIds), - ]); + if (configChangeAction) { + await updateAgent(soClient, esClient, agent.id, { + policy_revision: configChangeAction.policy_revision, + packages: configChangeAction?.ack_data?.packages, + }); + } + + if (agentActionsIds.length > 0) { + await soClient.bulkUpdate([ + ...buildUpdateAgentActionSentAt(agentActionsIds), + ]); + } return actions; } @@ -206,6 +200,7 @@ export interface AcksService { authenticateAgentWithAccessToken: ( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, request: KibanaRequest ) => Promise; diff --git a/x-pack/plugins/fleet/server/services/agents/actions.test.ts b/x-pack/plugins/fleet/server/services/agents/actions.test.ts index 5b3c2ea5ce7088..3d391cc89a7e31 100644 --- a/x-pack/plugins/fleet/server/services/agents/actions.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/actions.test.ts @@ -8,12 +8,12 @@ import { createAgentAction } from './actions'; import { SavedObject } from 'kibana/server'; import { AgentAction } from '../../../common/types/models'; -import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { savedObjectsClientMock, elasticsearchServiceMock } from 'src/core/server/mocks'; describe('test agent actions services', () => { it('should create a new action', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); - + const mockedEsClient = elasticsearchServiceMock.createInternalClient(); const newAgentAction: Omit = { agent_id: 'agentid', type: 'POLICY_CHANGE', @@ -32,7 +32,7 @@ describe('test agent actions services', () => { }, } as SavedObject) ); - await createAgentAction(mockSavedObjectsClient, newAgentAction); + await createAgentAction(mockSavedObjectsClient, mockedEsClient, newAgentAction); const createdAction = (mockSavedObjectsClient.create.mock .calls[0][1] as unknown) as AgentAction; diff --git a/x-pack/plugins/fleet/server/services/agents/actions.ts b/x-pack/plugins/fleet/server/services/agents/actions.ts index b45b7836eb46f9..8dfeac11dacf3b 100644 --- a/x-pack/plugins/fleet/server/services/agents/actions.ts +++ b/x-pack/plugins/fleet/server/services/agents/actions.ts @@ -13,8 +13,9 @@ import { BaseAgentActionSOAttributes, AgentActionSOAttributes, AgentPolicyActionSOAttributes, + FleetServerAgentAction, } from '../../../common/types/models'; -import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../../common/constants'; +import { AGENT_ACTION_SAVED_OBJECT_TYPE, AGENT_ACTIONS_INDEX } from '../../../common/constants'; import { isAgentActionSavedObject, isPolicyActionSavedObject, @@ -23,37 +24,45 @@ import { import { appContextService } from '../app_context'; import { nodeTypes } from '../../../../../../src/plugins/data/common'; +const ONE_MONTH_IN_MS = 2592000000; + export async function createAgentAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit ): Promise { - return createAction(soClient, newAgentAction); + return createAction(soClient, esClient, newAgentAction); } export async function bulkCreateAgentActions( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentActions: Array> ): Promise { - return bulkCreateActions(soClient, newAgentActions); + return bulkCreateActions(soClient, esClient, newAgentActions); } export function createAgentPolicyAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit ): Promise { - return createAction(soClient, newAgentAction); + return createAction(soClient, esClient, newAgentAction); } async function createAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit ): Promise; async function createAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit ): Promise; async function createAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit | Omit ): Promise { const actionSO = await soClient.create( @@ -65,6 +74,27 @@ async function createAction( } ); + if ( + appContextService.getConfig()?.agents?.fleetServerEnabled && + isAgentActionSavedObject(actionSO) + ) { + const body: FleetServerAgentAction = { + '@timestamp': new Date().toISOString(), + expiration: new Date(Date.now() + ONE_MONTH_IN_MS).toISOString(), + agents: [actionSO.attributes.agent_id], + action_id: actionSO.id, + data: newAgentAction.data, + type: newAgentAction.type, + }; + + await esClient.create({ + index: AGENT_ACTIONS_INDEX, + id: actionSO.id, + body, + refresh: 'wait_for', + }); + } + if (isAgentActionSavedObject(actionSO)) { const agentAction = savedObjectToAgentAction(actionSO); // Action `data` is encrypted, so is not returned from the saved object @@ -84,14 +114,17 @@ async function createAction( async function bulkCreateActions( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentActions: Array> ): Promise; async function bulkCreateActions( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentActions: Array> ): Promise; async function bulkCreateActions( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentActions: Array | Omit> ): Promise> { const { saved_objects: actionSOs } = await soClient.bulkCreate( @@ -105,6 +138,34 @@ async function bulkCreateActions( })) ); + if (appContextService.getConfig()?.agents?.fleetServerEnabled) { + await esClient.bulk({ + index: AGENT_ACTIONS_INDEX, + body: actionSOs.flatMap((actionSO) => { + if (!isAgentActionSavedObject(actionSO)) { + return []; + } + const body: FleetServerAgentAction = { + '@timestamp': new Date().toISOString(), + expiration: new Date(Date.now() + ONE_MONTH_IN_MS).toISOString(), + agents: [actionSO.attributes.agent_id], + action_id: actionSO.id, + data: actionSO.attributes.data ? JSON.parse(actionSO.attributes.data) : undefined, + type: actionSO.type, + }; + + return [ + { + create: { + _id: actionSO.id, + }, + }, + body, + ]; + }), + }); + } + return actionSOs.map((actionSO) => { if (isAgentActionSavedObject(actionSO)) { const agentAction = savedObjectToAgentAction(actionSO); @@ -316,6 +377,7 @@ export interface ActionsService { createAgentAction: ( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, newAgentAction: Omit ) => Promise; } diff --git a/x-pack/plugins/fleet/server/services/agents/authenticate.test.ts b/x-pack/plugins/fleet/server/services/agents/authenticate.test.ts index c59e2decebd992..5a1e86c15c0024 100644 --- a/x-pack/plugins/fleet/server/services/agents/authenticate.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/authenticate.test.ts @@ -6,10 +6,12 @@ */ import { KibanaRequest } from 'kibana/server'; -import { savedObjectsClientMock } from 'src/core/server/mocks'; +import { savedObjectsClientMock, elasticsearchServiceMock } from 'src/core/server/mocks'; import { authenticateAgentWithAccessToken } from './authenticate'; +const mockEsClient = elasticsearchServiceMock.createInternalClient(); + describe('test agent autenticate services', () => { it('should succeed with a valid API key and an active agent', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); @@ -32,7 +34,7 @@ describe('test agent autenticate services', () => { ], }) ); - await authenticateAgentWithAccessToken(mockSavedObjectsClient, { + await authenticateAgentWithAccessToken(mockSavedObjectsClient, mockEsClient, { auth: { isAuthenticated: true }, headers: { authorization: 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw==', @@ -62,7 +64,7 @@ describe('test agent autenticate services', () => { }) ); expect( - authenticateAgentWithAccessToken(mockSavedObjectsClient, { + authenticateAgentWithAccessToken(mockSavedObjectsClient, mockEsClient, { auth: { isAuthenticated: false }, headers: { authorization: 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw==', @@ -93,7 +95,7 @@ describe('test agent autenticate services', () => { }) ); expect( - authenticateAgentWithAccessToken(mockSavedObjectsClient, { + authenticateAgentWithAccessToken(mockSavedObjectsClient, mockEsClient, { auth: { isAuthenticated: true }, headers: { authorization: 'aaaa', @@ -124,7 +126,7 @@ describe('test agent autenticate services', () => { }) ); expect( - authenticateAgentWithAccessToken(mockSavedObjectsClient, { + authenticateAgentWithAccessToken(mockSavedObjectsClient, mockEsClient, { auth: { isAuthenticated: true }, headers: { authorization: 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw==', @@ -144,7 +146,7 @@ describe('test agent autenticate services', () => { }) ); expect( - authenticateAgentWithAccessToken(mockSavedObjectsClient, { + authenticateAgentWithAccessToken(mockSavedObjectsClient, mockEsClient, { auth: { isAuthenticated: true }, headers: { authorization: 'ApiKey cGVkVHVISUJURUR0OTN3VzBGaHI6TnU1U0JtbHJSeC12Rm9qQWpoSHlUZw==', diff --git a/x-pack/plugins/fleet/server/services/agents/authenticate.ts b/x-pack/plugins/fleet/server/services/agents/authenticate.ts index a773173b1ddc10..a03c35bdc6e737 100644 --- a/x-pack/plugins/fleet/server/services/agents/authenticate.ts +++ b/x-pack/plugins/fleet/server/services/agents/authenticate.ts @@ -6,13 +6,14 @@ */ import Boom from '@hapi/boom'; -import { KibanaRequest, SavedObjectsClientContract } from 'src/core/server'; +import { KibanaRequest, SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; import { Agent } from '../../types'; import * as APIKeyService from '../api_keys'; import { getAgentByAccessAPIKeyId } from './crud'; export async function authenticateAgentWithAccessToken( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, request: KibanaRequest ): Promise { if (!request.auth.isAuthenticated) { @@ -25,7 +26,7 @@ export async function authenticateAgentWithAccessToken( throw Boom.unauthorized(err.message); } - const agent = await getAgentByAccessAPIKeyId(soClient, res.apiKeyId); + const agent = await getAgentByAccessAPIKeyId(soClient, esClient, res.apiKeyId); return agent; } diff --git a/x-pack/plugins/fleet/server/services/agents/checkin/index.ts b/x-pack/plugins/fleet/server/services/agents/checkin/index.ts index 9a60abdc69423d..bcebedae2e07a1 100644 --- a/x-pack/plugins/fleet/server/services/agents/checkin/index.ts +++ b/x-pack/plugins/fleet/server/services/agents/checkin/index.ts @@ -19,9 +19,10 @@ import { AgentEventSOAttributes, } from '../../../types'; -import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants'; +import { AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../../constants'; import { agentCheckinState } from './state'; import { getAgentActionsForCheckin } from '../actions'; +import { updateAgent } from '../crud'; export async function agentCheckin( soClient: SavedObjectsClientContract, @@ -35,13 +36,7 @@ export async function agentCheckin( options?: { signal: AbortSignal } ) { const updateData: Partial = {}; - const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, data.events); - if ( - updatedErrorEvents && - !(updatedErrorEvents.length === 0 && agent.current_error_events.length === 0) - ) { - updateData.current_error_events = JSON.stringify(updatedErrorEvents); - } + await processEventsForCheckin(soClient, agent, data.events); if (data.localMetadata && !deepEqual(data.localMetadata, agent.local_metadata)) { updateData.local_metadata = data.localMetadata; } @@ -50,9 +45,8 @@ export async function agentCheckin( } // Update agent only if something changed if (Object.keys(updateData).length > 0) { - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agent.id, updateData); + await updateAgent(soClient, esClient, agent.id, updateData); } - // Check if some actions are not acknowledged let actions = await getAgentActionsForCheckin(soClient, agent.id); if (actions.length > 0) { diff --git a/x-pack/plugins/fleet/server/services/agents/checkin/state_connected_agents.ts b/x-pack/plugins/fleet/server/services/agents/checkin/state_connected_agents.ts index 83fd139a1e8e82..6156212a632032 100644 --- a/x-pack/plugins/fleet/server/services/agents/checkin/state_connected_agents.ts +++ b/x-pack/plugins/fleet/server/services/agents/checkin/state_connected_agents.ts @@ -5,10 +5,9 @@ * 2.0. */ -import { KibanaRequest, SavedObjectsBulkUpdateObject } from 'src/core/server'; +import { KibanaRequest } from 'src/core/server'; import { appContextService } from '../../app_context'; -import { AgentSOAttributes } from '../../../types'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../../constants'; +import { bulkUpdateAgents } from '../crud'; function getInternalUserSOClient() { const fakeRequest = ({ @@ -57,20 +56,17 @@ export function agentCheckinStateConnectedAgentsFactory() { if (agentToUpdate.size === 0) { return; } + const esClient = appContextService.getInternalUserESClient(); const internalSOClient = getInternalUserSOClient(); const now = new Date().toISOString(); - const updates: Array> = [ - ...agentToUpdate.values(), - ].map((agentId) => ({ - type: AGENT_SAVED_OBJECT_TYPE, - id: agentId, - attributes: { + const updates = [...agentToUpdate.values()].map((agentId) => ({ + agentId, + data: { last_checkin: now, }, })); - agentToUpdate = new Set([...connectedAgentsIds.values()]); - await internalSOClient.bulkUpdate(updates, { refresh: false }); + await bulkUpdateAgents(internalSOClient, esClient, updates); } return { diff --git a/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.test.ts b/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.test.ts index ca8378c117b7df..cd6e0ef61e3f08 100644 --- a/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { ElasticsearchClient } from 'kibana/server'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { take } from 'rxjs/operators'; import { @@ -17,6 +18,7 @@ import { outputType } from '../../../../common/constants'; jest.mock('../../app_context', () => ({ appContextService: { + getConfig: () => ({}), getInternalUserSOClient: () => { return {}; }, @@ -42,6 +44,8 @@ function getMockedNewActionSince() { return getNewActionsSince as jest.MockedFunction; } +const mockedEsClient = {} as ElasticsearchClient; + describe('test agent checkin new action services', () => { describe('newAgetActionObservable', () => { beforeEach(() => { @@ -161,12 +165,18 @@ describe('test agent checkin new action services', () => { ]; expect( - await createAgentActionFromPolicyAction(mockSavedObjectsClient, mockAgent, mockPolicyAction) + await createAgentActionFromPolicyAction( + mockSavedObjectsClient, + mockedEsClient, + mockAgent, + mockPolicyAction + ) ).toEqual(expectedResult); expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.10.0-SNAPSHOT' } } } }, mockPolicyAction ) @@ -175,6 +185,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.10.2' } } } }, mockPolicyAction ) @@ -183,6 +194,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '8.0.0' } } } }, mockPolicyAction ) @@ -191,6 +203,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '8.0.0-SNAPSHOT' } } } }, mockPolicyAction ) @@ -218,6 +231,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.9.0' } } } }, mockPolicyAction ) @@ -226,6 +240,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.9.3' } } } }, mockPolicyAction ) @@ -234,6 +249,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.9.1-SNAPSHOT' } } } }, mockPolicyAction ) @@ -242,6 +258,7 @@ describe('test agent checkin new action services', () => { expect( await createAgentActionFromPolicyAction( mockSavedObjectsClient, + mockedEsClient, { ...mockAgent, local_metadata: { elastic: { agent: { version: '7.8.2' } } } }, mockPolicyAction ) diff --git a/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.ts b/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.ts index 624b7bbcae5721..01759c2015cdf6 100644 --- a/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.ts +++ b/x-pack/plugins/fleet/server/services/agents/checkin/state_new_actions.ts @@ -45,7 +45,7 @@ import { } from '../actions'; import { appContextService } from '../../app_context'; import { toPromiseAbortable, AbortError, createRateLimiter } from './rxjs_utils'; -import { getAgent } from '../crud'; +import { getAgent, updateAgent } from '../crud'; function getInternalUserSOClient() { const fakeRequest = ({ @@ -106,31 +106,45 @@ function createAgentPolicyActionSharedObservable(agentPolicyId: string) { ); } -async function getOrCreateAgentDefaultOutputAPIKey( +async function getAgentDefaultOutputAPIKey( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agent: Agent -): Promise { - const { - attributes: { default_api_key: defaultApiKey }, - } = await appContextService - .getEncryptedSavedObjects() - .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, agent.id); +) { + if (appContextService.getConfig()?.agents?.fleetServerEnabled) { + return agent.default_api_key; + } else { + const { + attributes: { default_api_key: defaultApiKey }, + } = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, agent.id); - if (defaultApiKey) { return defaultApiKey; } +} + +async function getOrCreateAgentDefaultOutputAPIKey( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agent: Agent +): Promise { + const defaultAPIKey = await getAgentDefaultOutputAPIKey(soClient, esClient, agent); + if (defaultAPIKey) { + return defaultAPIKey; + } const outputAPIKey = await APIKeysService.generateOutputApiKey(soClient, 'default', agent.id); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agent.id, { + await updateAgent(soClient, esClient, agent.id, { default_api_key: outputAPIKey.key, default_api_key_id: outputAPIKey.id, }); - return outputAPIKey.key; } export async function createAgentActionFromPolicyAction( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agent: Agent, policyAction: AgentPolicyAction ) { @@ -168,7 +182,7 @@ export async function createAgentActionFromPolicyAction( ); // Mutate the policy to set the api token for this agent - const apiKey = await getOrCreateAgentDefaultOutputAPIKey(soClient, agent); + const apiKey = await getOrCreateAgentDefaultOutputAPIKey(soClient, esClient, agent); if (newAgentAction.data.policy) { newAgentAction.data.policy.outputs.default.api_key = apiKey; } @@ -249,7 +263,9 @@ export function agentCheckinStateNewActionsFactory() { (!agent.policy_revision || action.policy_revision > agent.policy_revision) ), rateLimiter(), - concatMap((policyAction) => createAgentActionFromPolicyAction(soClient, agent, policyAction)), + concatMap((policyAction) => + createAgentActionFromPolicyAction(soClient, esClient, agent, policyAction) + ), merge(newActions$), concatMap((data: AgentAction[] | undefined) => { if (data === undefined) { @@ -274,7 +290,7 @@ export function agentCheckinStateNewActionsFactory() { }), rateLimiter(), concatMap((policyAction) => - createAgentActionFromPolicyAction(soClient, agent, policyAction) + createAgentActionFromPolicyAction(soClient, esClient, agent, policyAction) ) ); } diff --git a/x-pack/plugins/fleet/server/services/agents/crud.ts b/x-pack/plugins/fleet/server/services/agents/crud.ts index 36506d05905958..c80fd77fc11ecc 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud.ts @@ -5,13 +5,8 @@ * 2.0. */ -import Boom from '@hapi/boom'; import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; - -import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; import { AgentSOAttributes, Agent, ListWithKuery } from '../../types'; -import { escapeSearchQueryPhrase } from '../saved_object'; -import { savedObjectToAgent } from './saved_objects'; import { appContextService, agentPolicyService } from '../../services'; import * as crudServiceSO from './crud_so'; import * as crudServiceFleetServer from './crud_fleet_server'; @@ -75,15 +70,15 @@ export async function getAgent( : crudServiceSO.getAgent(soClient, agentId); } -export async function getAgents(soClient: SavedObjectsClientContract, agentIds: string[]) { - const agentSOs = await soClient.bulkGet( - agentIds.map((agentId) => ({ - id: agentId, - type: AGENT_SAVED_OBJECT_TYPE, - })) - ); - const agents = agentSOs.saved_objects.map(savedObjectToAgent); - return agents; +export async function getAgents( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agentIds: string[] +) { + const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled; + return fleetServerEnabled + ? crudServiceFleetServer.getAgents(esClient, agentIds) + : crudServiceSO.getAgents(soClient, agentIds); } export async function getAgentPolicyForAgent( @@ -104,38 +99,39 @@ export async function getAgentPolicyForAgent( export async function getAgentByAccessAPIKeyId( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, accessAPIKeyId: string ): Promise { - const response = await soClient.find({ - type: AGENT_SAVED_OBJECT_TYPE, - searchFields: ['access_api_key_id'], - search: escapeSearchQueryPhrase(accessAPIKeyId), - }); - const [agent] = response.saved_objects.map(savedObjectToAgent); - - if (!agent) { - throw Boom.notFound('Agent not found'); - } - if (agent.access_api_key_id !== accessAPIKeyId) { - throw new Error('Agent api key id is not matching'); - } - if (!agent.active) { - throw Boom.forbidden('Agent inactive'); - } - - return agent; + const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled; + return fleetServerEnabled + ? crudServiceFleetServer.getAgentByAccessAPIKeyId(esClient, accessAPIKeyId) + : crudServiceSO.getAgentByAccessAPIKeyId(soClient, accessAPIKeyId); } export async function updateAgent( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agentId: string, - data: { - userProvidedMetatada: any; - } + data: Partial +) { + const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled; + return fleetServerEnabled + ? crudServiceFleetServer.updateAgent(esClient, agentId, data) + : crudServiceSO.updateAgent(soClient, agentId, data); +} + +export async function bulkUpdateAgents( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + data: Array<{ + agentId: string; + data: Partial; + }> ) { - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { - user_provided_metadata: data.userProvidedMetatada, - }); + const fleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled; + return fleetServerEnabled + ? crudServiceFleetServer.bulkUpdateAgents(esClient, data) + : crudServiceSO.bulkUpdateAgents(soClient, data); } export async function deleteAgent( diff --git a/x-pack/plugins/fleet/server/services/agents/crud_fleet_server.ts b/x-pack/plugins/fleet/server/services/agents/crud_fleet_server.ts index c9aa221edf4d29..caff15efff68c2 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud_fleet_server.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud_fleet_server.ts @@ -6,31 +6,46 @@ */ import Boom from '@hapi/boom'; -import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; +import { SearchResponse } from 'elasticsearch'; +import { ElasticsearchClient } from 'src/core/server'; -import { isAgentUpgradeable, SO_SEARCH_LIMIT } from '../../../common'; +import { FleetServerAgent, isAgentUpgradeable, SO_SEARCH_LIMIT } from '../../../common'; import { AGENT_SAVED_OBJECT_TYPE, AGENTS_INDEX } from '../../constants'; import { ESSearchHit } from '../../../../../typings/elasticsearch'; import { AgentSOAttributes, Agent, ListWithKuery } from '../../types'; import { escapeSearchQueryPhrase, normalizeKuery } from '../saved_object'; -import { savedObjectToAgent } from './saved_objects'; -import { searchHitToAgent } from './helpers'; +import { searchHitToAgent, agentSOAttributesToFleetServerAgentDoc } from './helpers'; import { appContextService } from '../../services'; +import { esKuery, KueryNode } from '../../../../../../src/plugins/data/server'; const ACTIVE_AGENT_CONDITION = 'active:true'; const INACTIVE_AGENT_CONDITION = `NOT (${ACTIVE_AGENT_CONDITION})`; -function _joinFilters(filters: string[], operator = 'AND') { - return filters.reduce((acc: string | undefined, filter) => { - if (acc) { - return `${acc} ${operator} (${filter})`; - } +function _joinFilters(filters: Array): KueryNode | undefined { + return filters + .filter((filter) => filter !== undefined) + .reduce((acc: KueryNode | undefined, kuery: string | KueryNode | undefined): + | KueryNode + | undefined => { + if (kuery === undefined) { + return acc; + } + const kueryNode: KueryNode = + typeof kuery === 'string' ? esKuery.fromKueryExpression(removeSOAttributes(kuery)) : kuery; - return `(${filter})`; - }, undefined); + if (!acc) { + return kueryNode; + } + + return { + type: 'function', + function: 'and', + arguments: [acc, kueryNode], + }; + }, undefined as KueryNode | undefined); } -function removeSOAttributes(kuery: string) { +export function removeSOAttributes(kuery: string) { return kuery.replace(/attributes\./g, '').replace(/fleet-agents\./g, ''); } @@ -57,20 +72,23 @@ export async function listAgents( const filters = []; if (kuery && kuery !== '') { - filters.push(removeSOAttributes(kuery)); + filters.push(kuery); } if (showInactive === false) { filters.push(ACTIVE_AGENT_CONDITION); } + const kueryNode = _joinFilters(filters); + const body = kueryNode ? { query: esKuery.toElasticsearchQuery(kueryNode) } : {}; + const res = await esClient.search({ index: AGENTS_INDEX, from: (page - 1) * perPage, size: perPage, sort: `${sortField}:${sortOrder}`, track_total_hits: true, - q: _joinFilters(filters), + body, }); let agentResults: Agent[] = res.body.hits.hits.map(searchHitToAgent); @@ -121,18 +139,20 @@ export async function countInactiveAgents( filters.push(normalizeKuery(AGENT_SAVED_OBJECT_TYPE, kuery)); } + const kueryNode = _joinFilters(filters); + const body = kueryNode ? { query: esKuery.toElasticsearchQuery(kueryNode) } : {}; + const res = await esClient.search({ index: AGENTS_INDEX, size: 0, track_total_hits: true, - q: _joinFilters(filters), + body, }); - return res.body.hits.total.value; } export async function getAgent(esClient: ElasticsearchClient, agentId: string) { - const agentHit = await esClient.get>({ + const agentHit = await esClient.get>({ index: AGENTS_INDEX, id: agentId, }); @@ -141,27 +161,31 @@ export async function getAgent(esClient: ElasticsearchClient, agentId: string) { return agent; } -export async function getAgents(soClient: SavedObjectsClientContract, agentIds: string[]) { - const agentSOs = await soClient.bulkGet( - agentIds.map((agentId) => ({ - id: agentId, - type: AGENT_SAVED_OBJECT_TYPE, - })) - ); - const agents = agentSOs.saved_objects.map(savedObjectToAgent); +export async function getAgents( + esClient: ElasticsearchClient, + agentIds: string[] +): Promise { + const body = { docs: agentIds.map((_id) => ({ _id })) }; + + const res = await esClient.mget({ + body, + index: AGENTS_INDEX, + }); + + const agents = res.body.docs.map(searchHitToAgent); return agents; } export async function getAgentByAccessAPIKeyId( - soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, accessAPIKeyId: string ): Promise { - const response = await soClient.find({ - type: AGENT_SAVED_OBJECT_TYPE, - searchFields: ['access_api_key_id'], - search: escapeSearchQueryPhrase(accessAPIKeyId), + const res = await esClient.search>({ + index: AGENTS_INDEX, + q: `access_api_key_id:${escapeSearchQueryPhrase(accessAPIKeyId)}`, }); - const [agent] = response.saved_objects.map(savedObjectToAgent); + + const [agent] = res.body.hits.hits.map(searchHitToAgent); if (!agent) { throw Boom.notFound('Agent not found'); @@ -177,15 +201,49 @@ export async function getAgentByAccessAPIKeyId( } export async function updateAgent( - soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agentId: string, - data: { - userProvidedMetatada: any; - } + data: Partial ) { - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { - user_provided_metadata: data.userProvidedMetatada, + await esClient.update({ + id: agentId, + index: AGENTS_INDEX, + body: { doc: agentSOAttributesToFleetServerAgentDoc(data) }, + refresh: 'wait_for', + }); +} + +export async function bulkUpdateAgents( + esClient: ElasticsearchClient, + updateData: Array<{ + agentId: string; + data: Partial; + }> +) { + const body = updateData.flatMap(({ agentId, data }) => [ + { + update: { + _id: agentId, + }, + }, + { + doc: { ...agentSOAttributesToFleetServerAgentDoc(data) }, + }, + ]); + + const res = await esClient.bulk({ + body, + index: AGENTS_INDEX, + refresh: 'wait_for', }); + + return { + items: res.body.items.map((item: { update: { _id: string; error?: Error } }) => ({ + id: item.update._id, + success: !item.update.error, + error: item.update.error, + })), + }; } export async function deleteAgent(esClient: ElasticsearchClient, agentId: string) { @@ -193,7 +251,7 @@ export async function deleteAgent(esClient: ElasticsearchClient, agentId: string id: agentId, index: AGENT_SAVED_OBJECT_TYPE, body: { - active: false, + doc: { active: false }, }, }); } diff --git a/x-pack/plugins/fleet/server/services/agents/crud_so.ts b/x-pack/plugins/fleet/server/services/agents/crud_so.ts index 11991a971829a0..c3ceb4b7502e26 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud_so.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud_so.ts @@ -6,7 +6,7 @@ */ import Boom from '@hapi/boom'; -import { SavedObjectsClientContract } from 'src/core/server'; +import { SavedObjectsBulkUpdateObject, SavedObjectsClientContract } from 'src/core/server'; import { isAgentUpgradeable } from '../../../common'; import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; @@ -197,13 +197,35 @@ export async function getAgentByAccessAPIKeyId( export async function updateAgent( soClient: SavedObjectsClientContract, agentId: string, - data: { - userProvidedMetatada: any; - } + data: Partial ) { - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { - user_provided_metadata: data.userProvidedMetatada, - }); + await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, data); +} + +export async function bulkUpdateAgents( + soClient: SavedObjectsClientContract, + updateData: Array<{ + agentId: string; + data: Partial; + }> +) { + const updates: Array> = updateData.map( + ({ agentId, data }) => ({ + type: AGENT_SAVED_OBJECT_TYPE, + id: agentId, + attributes: data, + }) + ); + + const res = await soClient.bulkUpdate(updates); + + return { + items: res.saved_objects.map((so) => ({ + id: so.id, + success: !so.error, + error: so.error, + })), + }; } export async function deleteAgent(soClient: SavedObjectsClientContract, agentId: string) { diff --git a/x-pack/plugins/fleet/server/services/agents/enroll.ts b/x-pack/plugins/fleet/server/services/agents/enroll.ts index b8be02af101b42..c984a84ceea014 100644 --- a/x-pack/plugins/fleet/server/services/agents/enroll.ts +++ b/x-pack/plugins/fleet/server/services/agents/enroll.ts @@ -6,14 +6,15 @@ */ import Boom from '@hapi/boom'; +import uuid from 'uuid/v4'; import semverParse from 'semver/functions/parse'; import semverDiff from 'semver/functions/diff'; import semverLte from 'semver/functions/lte'; import { SavedObjectsClientContract } from 'src/core/server'; -import { AgentType, Agent, AgentSOAttributes } from '../../types'; +import { AgentType, Agent, AgentSOAttributes, FleetServerAgent } from '../../types'; import { savedObjectToAgent } from './saved_objects'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; +import { AGENT_SAVED_OBJECT_TYPE, AGENTS_INDEX } from '../../constants'; import * as APIKeyService from '../api_keys'; import { appContextService } from '../app_context'; @@ -26,6 +27,36 @@ export async function enroll( const agentVersion = metadata?.local?.elastic?.agent?.version; validateAgentVersion(agentVersion); + if (appContextService.getConfig()?.agents?.fleetServerEnabled) { + const esClient = appContextService.getInternalUserESClient(); + + const agentId = uuid(); + const accessAPIKey = await APIKeyService.generateAccessApiKey(soClient, agentId); + const fleetServerAgent: FleetServerAgent = { + active: true, + policy_id: agentPolicyId, + type, + enrolled_at: new Date().toISOString(), + user_provided_metadata: metadata?.userProvided ?? {}, + local_metadata: metadata?.local ?? {}, + access_api_key_id: accessAPIKey.id, + }; + await esClient.create({ + index: AGENTS_INDEX, + body: fleetServerAgent, + id: agentId, + refresh: 'wait_for', + }); + + return { + id: agentId, + current_error_events: [], + packages: [], + ...fleetServerAgent, + access_api_key: accessAPIKey.key, + } as Agent; + } + const agentData: AgentSOAttributes = { active: true, policy_id: agentPolicyId, diff --git a/x-pack/plugins/fleet/server/services/agents/helpers.ts b/x-pack/plugins/fleet/server/services/agents/helpers.ts index 1000a1b1459328..90d85e98ecd679 100644 --- a/x-pack/plugins/fleet/server/services/agents/helpers.ts +++ b/x-pack/plugins/fleet/server/services/agents/helpers.ts @@ -6,17 +6,30 @@ */ import { ESSearchHit } from '../../../../../typings/elasticsearch'; -import { Agent, AgentSOAttributes } from '../../types'; +import { Agent, AgentSOAttributes, FleetServerAgent } from '../../types'; -export function searchHitToAgent(hit: ESSearchHit): Agent { +export function searchHitToAgent(hit: ESSearchHit): Agent { return { id: hit._id, ...hit._source, - current_error_events: hit._source.current_error_events - ? JSON.parse(hit._source.current_error_events) - : [], + policy_revision: hit._source.policy_revision_idx, + current_error_events: [], access_api_key: undefined, status: undefined, packages: hit._source.packages ?? [], }; } + +export function agentSOAttributesToFleetServerAgentDoc( + data: Partial +): Partial> { + const { policy_revision: policyRevison, ...rest } = data; + + const doc: Partial> = { ...rest }; + + if (policyRevison !== undefined) { + doc.policy_revision_idx = policyRevison; + } + + return doc; +} diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts index 7338c440483ea6..466870bead71ce 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts @@ -107,6 +107,9 @@ function createClientMock() { saved_objects: [await soClientMock.create(type, attributes)], }; }); + soClientMock.bulkUpdate.mockResolvedValue({ + saved_objects: [], + }); soClientMock.get.mockImplementation(async (_, id) => { switch (id) { diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.ts b/x-pack/plugins/fleet/server/services/agents/reassign.ts index 9f4373ab553ecf..62d59aada3b7ba 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.ts @@ -7,11 +7,16 @@ import type { SavedObjectsClientContract, ElasticsearchClient } from 'kibana/server'; import Boom from '@hapi/boom'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; -import type { AgentSOAttributes } from '../../types'; -import { AgentReassignmentError } from '../../errors'; import { agentPolicyService } from '../agent_policy'; -import { getAgentPolicyForAgent, getAgents, listAllAgents } from './crud'; +import { + getAgents, + getAgentPolicyForAgent, + listAllAgents, + updateAgent, + bulkUpdateAgents, +} from './crud'; +import { AgentReassignmentError } from '../../errors'; + import { createAgentAction, bulkCreateAgentActions } from './actions'; export async function reassignAgent( @@ -27,12 +32,12 @@ export async function reassignAgent( await reassignAgentIsAllowed(soClient, esClient, agentId, newAgentPolicyId); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { + await updateAgent(soClient, esClient, agentId, { policy_id: newAgentPolicyId, policy_revision: null, }); - await createAgentAction(soClient, { + await createAgentAction(soClient, esClient, { agent_id: agentId, created_at: new Date().toISOString(), type: 'INTERNAL_POLICY_REASSIGN', @@ -73,7 +78,7 @@ export async function reassignAgents( kuery: string; }, newAgentPolicyId: string -) { +): Promise<{ items: Array<{ id: string; sucess: boolean; error?: Error }> }> { const agentPolicy = await agentPolicyService.get(soClient, newAgentPolicyId); if (!agentPolicy) { throw Boom.notFound(`Agent policy not found: ${newAgentPolicyId}`); @@ -82,7 +87,7 @@ export async function reassignAgents( // Filter to agents that do not already use the new agent policy ID const agents = 'agentIds' in options - ? await getAgents(soClient, options.agentIds) + ? await getAgents(soClient, esClient, options.agentIds) : ( await listAllAgents(soClient, esClient, { kuery: options.kuery, @@ -99,20 +104,22 @@ export async function reassignAgents( (agent, index) => settled[index].status === 'fulfilled' && agent.policy_id !== newAgentPolicyId ); - // Update the necessary agents - const res = await soClient.bulkUpdate( + const res = await bulkUpdateAgents( + soClient, + esClient, agentsToUpdate.map((agent) => ({ - type: AGENT_SAVED_OBJECT_TYPE, - id: agent.id, - attributes: { + agentId: agent.id, + data: { policy_id: newAgentPolicyId, policy_revision: null, }, })) ); + const now = new Date().toISOString(); await bulkCreateAgentActions( soClient, + esClient, agentsToUpdate.map((agent) => ({ agent_id: agent.id, created_at: now, diff --git a/x-pack/plugins/fleet/server/services/agents/status.ts b/x-pack/plugins/fleet/server/services/agents/status.ts index c75b91b3fbd11c..42d3aff2b0d702 100644 --- a/x-pack/plugins/fleet/server/services/agents/status.ts +++ b/x-pack/plugins/fleet/server/services/agents/status.ts @@ -14,6 +14,8 @@ import { AgentStatus } from '../../types'; import { AgentStatusKueryHelper } from '../../../common/services'; import { esKuery, KueryNode } from '../../../../../../src/plugins/data/server'; import { normalizeKuery } from '../saved_object'; +import { appContextService } from '../app_context'; +import { removeSOAttributes } from './crud_fleet_server'; export async function getAgentStatusById( soClient: SavedObjectsClientContract, @@ -27,6 +29,8 @@ export async function getAgentStatusById( export const getAgentStatus = AgentStatusKueryHelper.getAgentStatus; function joinKuerys(...kuerys: Array) { + const isFleetServerEnabled = appContextService.getConfig()?.agents?.fleetServerEnabled; + return kuerys .filter((kuery) => kuery !== undefined) .reduce((acc: KueryNode | undefined, kuery: string | undefined): KueryNode | undefined => { @@ -34,7 +38,9 @@ function joinKuerys(...kuerys: Array) { return acc; } const normalizedKuery: KueryNode = esKuery.fromKueryExpression( - normalizeKuery(AGENT_SAVED_OBJECT_TYPE, kuery || '') + isFleetServerEnabled + ? removeSOAttributes(kuery || '') + : normalizeKuery(AGENT_SAVED_OBJECT_TYPE, kuery || '') ); if (!acc) { diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts index b8c1b7befb443c..cd46cff0f8a174 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts @@ -73,6 +73,7 @@ describe('unenrollAgents (plural)', () => { }); it('cannot unenroll from a managed policy', async () => { const soClient = createClientMock(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; const idsToUnenroll = [agentInUnmanagedSO.id, agentInManagedSO.id, agentInUnmanagedSO2.id]; await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); @@ -98,6 +99,9 @@ function createClientMock() { saved_objects: [await soClientMock.create(type, attributes)], }; }); + soClientMock.bulkUpdate.mockResolvedValue({ + saved_objects: [], + }); soClientMock.get.mockImplementation(async (_, id) => { switch (id) { diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.ts index e2fa83cf32b637..72d551a1229801 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.ts @@ -4,13 +4,19 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import type { AgentSOAttributes } from '../../types'; -import { AgentUnenrollmentError } from '../../errors'; -import { AGENT_SAVED_OBJECT_TYPE } from '../../constants'; + +import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; import * as APIKeyService from '../api_keys'; import { createAgentAction, bulkCreateAgentActions } from './actions'; -import { getAgent, getAgentPolicyForAgent, getAgents, listAllAgents } from './crud'; +import { + getAgent, + updateAgent, + getAgentPolicyForAgent, + getAgents, + listAllAgents, + bulkUpdateAgents, +} from './crud'; +import { AgentUnenrollmentError } from '../../errors'; async function unenrollAgentIsAllowed( soClient: SavedObjectsClientContract, @@ -35,12 +41,12 @@ export async function unenrollAgent( await unenrollAgentIsAllowed(soClient, esClient, agentId); const now = new Date().toISOString(); - await createAgentAction(soClient, { + await createAgentAction(soClient, esClient, { agent_id: agentId, created_at: now, type: 'UNENROLL', }); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { + await updateAgent(soClient, esClient, agentId, { unenrollment_started_at: now, }); } @@ -58,7 +64,7 @@ export async function unenrollAgents( ) { const agents = 'agentIds' in options - ? await getAgents(soClient, options.agentIds) + ? await getAgents(soClient, esClient, options.agentIds) : ( await listAllAgents(soClient, esClient, { kuery: options.kuery, @@ -83,6 +89,7 @@ export async function unenrollAgents( // Create unenroll action for each agent await bulkCreateAgentActions( soClient, + esClient, agentsToUpdate.map((agent) => ({ agent_id: agent.id, created_at: now, @@ -91,11 +98,12 @@ export async function unenrollAgents( ); // Update the necessary agents - return await soClient.bulkUpdate( + return bulkUpdateAgents( + soClient, + esClient, agentsToUpdate.map((agent) => ({ - type: AGENT_SAVED_OBJECT_TYPE, - id: agent.id, - attributes: { + agentId: agent.id, + data: { unenrollment_started_at: now, }, })) @@ -118,7 +126,7 @@ export async function forceUnenrollAgent( : undefined, ]); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { + await updateAgent(soClient, esClient, agentId, { active: false, unenrolled_at: new Date().toISOString(), }); @@ -138,7 +146,7 @@ export async function forceUnenrollAgents( // Filter to agents that are not already unenrolled const agents = 'agentIds' in options - ? await getAgents(soClient, options.agentIds) + ? await getAgents(soClient, esClient, options.agentIds) : ( await listAllAgents(soClient, esClient, { kuery: options.kuery, @@ -163,13 +171,13 @@ export async function forceUnenrollAgents( if (apiKeys.length) { APIKeyService.invalidateAPIKeys(soClient, apiKeys); } - // Update the necessary agents - return await soClient.bulkUpdate( + return bulkUpdateAgents( + soClient, + esClient, agentsToUpdate.map((agent) => ({ - type: AGENT_SAVED_OBJECT_TYPE, - id: agent.id, - attributes: { + agentId: agent.id, + data: { active: false, unenrolled_at: now, }, diff --git a/x-pack/plugins/fleet/server/services/agents/upgrade.ts b/x-pack/plugins/fleet/server/services/agents/upgrade.ts index 7475ad49681427..5105e145309827 100644 --- a/x-pack/plugins/fleet/server/services/agents/upgrade.ts +++ b/x-pack/plugins/fleet/server/services/agents/upgrade.ts @@ -6,20 +6,22 @@ */ import { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server'; -import { AgentSOAttributes, AgentAction, AgentActionSOAttributes } from '../../types'; -import { AGENT_ACTION_SAVED_OBJECT_TYPE, AGENT_SAVED_OBJECT_TYPE } from '../../constants'; +import { AgentAction, AgentActionSOAttributes } from '../../types'; +import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../constants'; import { bulkCreateAgentActions, createAgentAction } from './actions'; -import { getAgents, listAllAgents } from './crud'; +import { getAgents, listAllAgents, updateAgent, bulkUpdateAgents } from './crud'; import { isAgentUpgradeable } from '../../../common/services'; import { appContextService } from '../app_context'; export async function sendUpgradeAgentAction({ soClient, + esClient, agentId, version, sourceUri, }: { soClient: SavedObjectsClientContract; + esClient: ElasticsearchClient; agentId: string; version: string; sourceUri: string | undefined; @@ -29,21 +31,22 @@ export async function sendUpgradeAgentAction({ version, source_uri: sourceUri, }; - await createAgentAction(soClient, { + await createAgentAction(soClient, esClient, { agent_id: agentId, created_at: now, data, ack_data: data, type: 'UPGRADE', }); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentId, { - upgraded_at: undefined, + await updateAgent(soClient, esClient, agentId, { + upgraded_at: null, upgrade_started_at: now, }); } export async function ackAgentUpgraded( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, agentAction: AgentAction ) { const { @@ -52,9 +55,9 @@ export async function ackAgentUpgraded( if (!ackData) throw new Error('data missing from UPGRADE action'); const { version } = JSON.parse(ackData); if (!version) throw new Error('version missing from UPGRADE action'); - await soClient.update(AGENT_SAVED_OBJECT_TYPE, agentAction.agent_id, { + await updateAgent(soClient, esClient, agentAction.agent_id, { upgraded_at: new Date().toISOString(), - upgrade_started_at: undefined, + upgrade_started_at: null, }); } @@ -79,7 +82,7 @@ export async function sendUpgradeAgentsActions( // Filter out agents currently unenrolling, agents unenrolled, and agents not upgradeable const agents = 'agentIds' in options - ? await getAgents(soClient, options.agentIds) + ? await getAgents(soClient, esClient, options.agentIds) : ( await listAllAgents(soClient, esClient, { kuery: options.kuery, @@ -97,6 +100,7 @@ export async function sendUpgradeAgentsActions( // Create upgrade action for each agent await bulkCreateAgentActions( soClient, + esClient, agentsToUpdate.map((agent) => ({ agent_id: agent.id, created_at: now, @@ -106,12 +110,13 @@ export async function sendUpgradeAgentsActions( })) ); - return await soClient.bulkUpdate( + return await bulkUpdateAgents( + soClient, + esClient, agentsToUpdate.map((agent) => ({ - type: AGENT_SAVED_OBJECT_TYPE, - id: agent.id, - attributes: { - upgraded_at: undefined, + agentId: agent.id, + data: { + upgraded_at: null, upgrade_started_at: now, }, })) diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts index 97d48702cf4c6e..85812fee3885c8 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts @@ -79,3 +79,15 @@ export async function generateEnrollmentAPIKey( return enrollmentApiKeyServiceSO.generateEnrollmentAPIKey(soClient, data); } } + +export async function getEnrollmentAPIKeyById( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + apiKeyId: string +) { + if (appContextService.getConfig()?.agents?.fleetServerEnabled === true) { + return enrollmentApiKeyServiceFleetServer.getEnrollmentAPIKeyById(esClient, apiKeyId); + } else { + return enrollmentApiKeyServiceSO.getEnrollmentAPIKeyById(soClient, apiKeyId); + } +} diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_fleet_server.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_fleet_server.ts index d42cb19a340bdd..f5d0015297daa0 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_fleet_server.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_fleet_server.ts @@ -7,41 +7,15 @@ import uuid from 'uuid'; import Boom from '@hapi/boom'; +import { GetResponse } from 'elasticsearch'; import { ResponseError } from '@elastic/elasticsearch/lib/errors'; import { SavedObjectsClientContract, ElasticsearchClient } from 'src/core/server'; +import { ESSearchResponse as SearchResponse } from '../../../../../typings/elasticsearch'; import { EnrollmentAPIKey, FleetServerEnrollmentAPIKey } from '../../types'; import { ENROLLMENT_API_KEYS_INDEX } from '../../constants'; import { createAPIKey, invalidateAPIKeys } from './security'; import { agentPolicyService } from '../agent_policy'; - -// TODO Move these types to another file -interface SearchResponse { - took: number; - timed_out: boolean; - _scroll_id?: string; - hits: { - total: { - value: number; - relation: string; - }; - max_score: number; - hits: Array<{ - _index: string; - _type: string; - _id: string; - _score: number; - _source: T; - _version?: number; - fields?: any; - highlight?: any; - inner_hits?: any; - matched_queries?: string[]; - sort?: string[]; - }>; - }; -} - -type SearchHit = SearchResponse['hits']['hits'][0]; +import { escapeSearchQueryPhrase } from '../saved_object'; export async function listEnrollmentApiKeys( esClient: ElasticsearchClient, @@ -54,7 +28,7 @@ export async function listEnrollmentApiKeys( ): Promise<{ items: EnrollmentAPIKey[]; total: any; page: any; perPage: any }> { const { page = 1, perPage = 20, kuery } = options; - const res = await esClient.search>({ + const res = await esClient.search>({ index: ENROLLMENT_API_KEYS_INDEX, from: (page - 1) * perPage, size: perPage, @@ -78,7 +52,7 @@ export async function getEnrollmentAPIKey( id: string ): Promise { try { - const res = await esClient.get>({ + const res = await esClient.get>({ index: ENROLLMENT_API_KEYS_INDEX, id, }); @@ -185,6 +159,21 @@ export async function generateEnrollmentAPIKey( }; } +export async function getEnrollmentAPIKeyById(esClient: ElasticsearchClient, apiKeyId: string) { + const res = await esClient.search>({ + index: ENROLLMENT_API_KEYS_INDEX, + q: `api_key_id:${escapeSearchQueryPhrase(apiKeyId)}`, + }); + + const [enrollmentAPIKey] = res.body.hits.hits.map(esDocToEnrollmentApiKey); + + if (enrollmentAPIKey?.api_key_id !== apiKeyId) { + throw new Error('find enrollmentKeyById returned an incorrect key'); + } + + return enrollmentAPIKey; +} + async function validateAgentPolicyId(soClient: SavedObjectsClientContract, agentPolicyId: string) { try { await agentPolicyService.get(soClient, agentPolicyId); @@ -196,7 +185,10 @@ async function validateAgentPolicyId(soClient: SavedObjectsClientContract, agent } } -function esDocToEnrollmentApiKey(doc: SearchHit): EnrollmentAPIKey { +function esDocToEnrollmentApiKey(doc: { + _id: string; + _source: FleetServerEnrollmentAPIKey; +}): EnrollmentAPIKey { return { id: doc._id, ...doc._source, diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_so.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_so.ts index b3beab546c811b..014bc58e747ea4 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_so.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key_so.ts @@ -13,7 +13,7 @@ import { ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE } from '../../constants'; import { createAPIKey, invalidateAPIKeys } from './security'; import { agentPolicyService } from '../agent_policy'; import { appContextService } from '../app_context'; -import { normalizeKuery } from '../saved_object'; +import { normalizeKuery, escapeSearchQueryPhrase } from '../saved_object'; export async function listEnrollmentApiKeys( soClient: SavedObjectsClientContract, @@ -159,6 +159,25 @@ async function validateAgentPolicyId(soClient: SavedObjectsClientContract, agent } } +export async function getEnrollmentAPIKeyById( + soClient: SavedObjectsClientContract, + apiKeyId: string +) { + const [enrollmentAPIKey] = ( + await soClient.find({ + type: ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + searchFields: ['api_key_id'], + search: escapeSearchQueryPhrase(apiKeyId), + }) + ).saved_objects.map(savedObjectToEnrollmentApiKey); + + if (enrollmentAPIKey?.api_key_id !== apiKeyId) { + throw new Error('find enrollmentKeyById returned an incorrect key'); + } + + return enrollmentAPIKey; +} + function savedObjectToEnrollmentApiKey({ error, attributes, diff --git a/x-pack/plugins/fleet/server/services/api_keys/index.ts b/x-pack/plugins/fleet/server/services/api_keys/index.ts index 5cdadeb0c82d84..65051163c78c3a 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/index.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/index.ts @@ -5,11 +5,8 @@ * 2.0. */ -import { SavedObjectsClientContract, SavedObject, KibanaRequest } from 'src/core/server'; -import { ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE } from '../../constants'; -import { EnrollmentAPIKeySOAttributes, EnrollmentAPIKey } from '../../types'; +import { SavedObjectsClientContract, KibanaRequest } from 'src/core/server'; import { createAPIKey } from './security'; -import { escapeSearchQueryPhrase } from '../saved_object'; export { invalidateAPIKeys } from './security'; export * from './enrollment_api_key'; @@ -70,25 +67,6 @@ export async function generateAccessApiKey(soClient: SavedObjectsClientContract, return { id: key.id, key: Buffer.from(`${key.id}:${key.api_key}`).toString('base64') }; } -export async function getEnrollmentAPIKeyById( - soClient: SavedObjectsClientContract, - apiKeyId: string -) { - const [enrollmentAPIKey] = ( - await soClient.find({ - type: ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, - searchFields: ['api_key_id'], - search: escapeSearchQueryPhrase(apiKeyId), - }) - ).saved_objects.map(_savedObjectToEnrollmentApiKey); - - if (enrollmentAPIKey?.api_key_id !== apiKeyId) { - throw new Error('find enrollmentKeyById returned an incorrect key'); - } - - return enrollmentAPIKey; -} - export function parseApiKeyFromHeaders(headers: KibanaRequest['headers']) { const authorizationHeader = headers.authorization; @@ -117,18 +95,3 @@ export function parseApiKey(apiKey: string) { apiKeyId, }; } - -function _savedObjectToEnrollmentApiKey({ - error, - attributes, - id, -}: SavedObject): EnrollmentAPIKey { - if (error) { - throw new Error(error.message); - } - - return { - id, - ...attributes, - }; -} diff --git a/x-pack/plugins/fleet/server/services/fleet_server_migration.ts b/x-pack/plugins/fleet/server/services/fleet_server_migration.ts index f982332886e943..170bec54983c0e 100644 --- a/x-pack/plugins/fleet/server/services/fleet_server_migration.ts +++ b/x-pack/plugins/fleet/server/services/fleet_server_migration.ts @@ -5,11 +5,18 @@ * 2.0. */ +import { isBoom } from '@hapi/boom'; import { KibanaRequest } from 'src/core/server'; import { ENROLLMENT_API_KEYS_INDEX, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + AGENT_POLICY_INDEX, + AGENTS_INDEX, FleetServerEnrollmentAPIKey, + AGENT_SAVED_OBJECT_TYPE, + AgentSOAttributes, + FleetServerAgent, + SO_SEARCH_LIMIT, FLEET_SERVER_PACKAGE, FLEET_SERVER_INDICES, } from '../../common'; @@ -17,6 +24,9 @@ import { listEnrollmentApiKeys, getEnrollmentAPIKey } from './api_keys/enrollmen import { appContextService } from './app_context'; import { getInstallation } from './epm/packages'; +import { isAgentsSetup } from './agents'; +import { agentPolicyService } from './agent_policy'; + export async function isFleetServerSetup() { const pkgInstall = await getInstallation({ savedObjectsClient: getInternalUserSOClient(), @@ -28,7 +38,6 @@ export async function isFleetServerSetup() { } const esClient = appContextService.getInternalUserESClient(); - const exists = await Promise.all( FLEET_SERVER_INDICES.map(async (index) => { const res = await esClient.indices.exists({ @@ -42,7 +51,11 @@ export async function isFleetServerSetup() { } export async function runFleetServerMigration() { - await migrateEnrollmentApiKeys(); + // If Agents are not setup skip as there is nothing to migrate + if (!(await isAgentsSetup(getInternalUserSOClient()))) { + return; + } + await Promise.all([migrateEnrollmentApiKeys(), migrateAgentPolicies(), migrateAgents()]); } function getInternalUserSOClient() { @@ -64,6 +77,65 @@ function getInternalUserSOClient() { return appContextService.getInternalUserSOClient(fakeRequest); } +async function migrateAgents() { + const esClient = appContextService.getInternalUserESClient(); + const soClient = getInternalUserSOClient(); + let hasMore = true; + while (hasMore) { + const res = await soClient.find({ + type: AGENT_SAVED_OBJECT_TYPE, + page: 1, + perPage: 100, + }); + + if (res.total === 0) { + hasMore = false; + } + for (const so of res.saved_objects) { + try { + const { + attributes, + } = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, so.id); + + const body: FleetServerAgent = { + type: attributes.type, + active: attributes.active, + enrolled_at: attributes.enrolled_at, + unenrolled_at: attributes.unenrolled_at, + unenrollment_started_at: attributes.unenrollment_started_at, + upgraded_at: attributes.upgraded_at, + upgrade_started_at: attributes.upgrade_started_at, + access_api_key_id: attributes.access_api_key_id, + user_provided_metadata: attributes.user_provided_metadata, + local_metadata: attributes.local_metadata, + policy_id: attributes.policy_id, + policy_revision_idx: attributes.policy_revision || undefined, + last_checkin: attributes.last_checkin, + last_checkin_status: attributes.last_checkin_status, + default_api_key_id: attributes.default_api_key_id, + default_api_key: attributes.default_api_key, + packages: attributes.packages, + }; + await esClient.create({ + index: AGENTS_INDEX, + body, + id: so.id, + refresh: 'wait_for', + }); + + await soClient.delete(AGENT_SAVED_OBJECT_TYPE, so.id); + } catch (error) { + // swallow 404 error has multiple Kibana can run the migration at the same time + if (!is404Error(error)) { + throw error; + } + } + } + } +} + async function migrateEnrollmentApiKeys() { const esClient = appContextService.getInternalUserESClient(); const soClient = getInternalUserSOClient(); @@ -77,24 +149,61 @@ async function migrateEnrollmentApiKeys() { hasMore = false; } for (const item of res.items) { - const key = await getEnrollmentAPIKey(soClient, item.id); - - const body: FleetServerEnrollmentAPIKey = { - api_key: key.api_key, - api_key_id: key.api_key_id, - active: key.active, - created_at: key.created_at, - name: key.name, - policy_id: key.policy_id, - }; - await esClient.create({ - index: ENROLLMENT_API_KEYS_INDEX, - body, - id: key.id, - refresh: 'wait_for', - }); + try { + const key = await getEnrollmentAPIKey(soClient, item.id); + + const body: FleetServerEnrollmentAPIKey = { + api_key: key.api_key, + api_key_id: key.api_key_id, + active: key.active, + created_at: key.created_at, + name: key.name, + policy_id: key.policy_id, + }; + await esClient.create({ + index: ENROLLMENT_API_KEYS_INDEX, + body, + id: key.id, + refresh: 'wait_for', + }); - await soClient.delete(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, key.id); + await soClient.delete(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, key.id); + } catch (error) { + // swallow 404 error has multiple Kibana can run the migration at the same time + if (!is404Error(error)) { + throw error; + } + } } } } + +async function migrateAgentPolicies() { + const esClient = appContextService.getInternalUserESClient(); + const soClient = getInternalUserSOClient(); + const { items: agentPolicies } = await agentPolicyService.list(soClient, { + perPage: SO_SEARCH_LIMIT, + }); + + await Promise.all( + agentPolicies.map(async (agentPolicy) => { + const res = await esClient.search({ + index: AGENT_POLICY_INDEX, + q: `policy_id:${agentPolicy.id}`, + track_total_hits: true, + }); + + if (res.body.hits.total.value === 0) { + return agentPolicyService.createFleetPolicyChangeFleetServer( + soClient, + esClient, + agentPolicy.id + ); + } + }) + ); +} + +function is404Error(error: any) { + return isBoom(error) && error.output.statusCode === 404; +} diff --git a/x-pack/plugins/fleet/server/services/index.ts b/x-pack/plugins/fleet/server/services/index.ts index 9999ab91e31b22..77ce882275b6bf 100644 --- a/x-pack/plugins/fleet/server/services/index.ts +++ b/x-pack/plugins/fleet/server/services/index.ts @@ -49,6 +49,7 @@ export interface AgentService { */ authenticateAgentWithAccessToken( soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, request: KibanaRequest ): Promise; /** diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index ab24c26e0cdfae..f19ad4e7fe417d 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -93,6 +93,16 @@ async function createSetupSideEffects( await runFleetServerMigration(); } + if (appContextService.getConfig()?.agents?.fleetServerEnabled) { + await ensureInstalledPackage({ + savedObjectsClient: soClient, + pkgName: FLEET_SERVER_PACKAGE, + callCluster, + }); + await ensureFleetServerIndicesCreated(esClient); + await runFleetServerMigration(); + } + // If we just created the default policy, ensure default packages are added to it if (defaultAgentPolicyCreated) { const agentPolicyWithPackagePolicies = await agentPolicyService.get( diff --git a/x-pack/plugins/fleet/server/types/index.tsx b/x-pack/plugins/fleet/server/types/index.tsx index 0c35fc29e01cd7..fda1568c56e0e6 100644 --- a/x-pack/plugins/fleet/server/types/index.tsx +++ b/x-pack/plugins/fleet/server/types/index.tsx @@ -81,6 +81,9 @@ export { dataTypes, // Fleet Server types FleetServerEnrollmentAPIKey, + FleetServerAgent, + FleetServerAgentAction, + FleetServerPolicy, } from '../../common'; export type CallESAsCurrentUser = LegacyScopedClusterClient['callAsCurrentUser']; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 6b58ca71f7f4e7..a2aff41b68df70 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -174,6 +174,9 @@ describe('test alerts route', () => { savedObjects: { client: mockSavedObjectClient, }, + elasticsearch: { + client: { asInternalUser: elasticsearchServiceMock.createInternalClient() }, + }, }, } as unknown) as SecuritySolutionRequestHandlerContext, mockRequest, @@ -218,6 +221,9 @@ describe('test alerts route', () => { savedObjects: { client: mockSavedObjectClient, }, + elasticsearch: { + client: { asInternalUser: elasticsearchServiceMock.createInternalClient() }, + }, }, } as unknown) as SecuritySolutionRequestHandlerContext, mockRequest, @@ -252,6 +258,9 @@ describe('test alerts route', () => { savedObjects: { client: mockSavedObjectClient, }, + elasticsearch: { + client: { asInternalUser: elasticsearchServiceMock.createInternalClient() }, + }, }, } as unknown) as SecuritySolutionRequestHandlerContext, mockRequest, @@ -280,6 +289,9 @@ describe('test alerts route', () => { savedObjects: { client: mockSavedObjectClient, }, + elasticsearch: { + client: { asInternalUser: elasticsearchServiceMock.createInternalClient() }, + }, }, } as unknown) as SecuritySolutionRequestHandlerContext, mockRequest, @@ -314,6 +326,9 @@ describe('test alerts route', () => { savedObjects: { client: mockSavedObjectClient, }, + elasticsearch: { + client: { asInternalUser: elasticsearchServiceMock.createInternalClient() }, + }, }, } as unknown) as SecuritySolutionRequestHandlerContext, mockRequest, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 95563c7c48ef58..3dbaa137bb9281 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -54,7 +54,11 @@ export function registerDownloadExceptionListRoute( // The ApiKey must be associated with an enrolled Fleet agent try { scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); - await authenticateAgentWithAccessToken(scopedSOClient, req); + await authenticateAgentWithAccessToken( + scopedSOClient, + context.core.elasticsearch.client.asInternalUser, + req + ); } catch (err) { if ((err.isBoom ? err.output.statusCode : err.statusCode) === 401) { return res.unauthorized(); diff --git a/x-pack/test/accessibility/apps/uptime.ts b/x-pack/test/accessibility/apps/uptime.ts index ec1f37ca02be2f..d7a9cfc0d08b40 100644 --- a/x-pack/test/accessibility/apps/uptime.ts +++ b/x-pack/test/accessibility/apps/uptime.ts @@ -18,7 +18,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const es = getService('es'); - describe('uptime', () => { + // FLAKY: https://github.com/elastic/kibana/issues/90555 + describe.skip('uptime', () => { before(async () => { await esArchiver.load('uptime/blank'); await makeChecks(es, A11Y_TEST_MONITOR_ID, 150, 1, 1000, { diff --git a/x-pack/test/api_integration/apis/management/index_management/indices.js b/x-pack/test/api_integration/apis/management/index_management/indices.js index cef1bdbba754b8..3653d9916466d0 100644 --- a/x-pack/test/api_integration/apis/management/index_management/indices.js +++ b/x-pack/test/api_integration/apis/management/index_management/indices.js @@ -34,7 +34,8 @@ export default function ({ getService }) { clearCache, } = registerHelpers({ supertest }); - describe('indices', () => { + // Failing: See https://github.com/elastic/kibana/issues/64473 + describe.skip('indices', () => { after(() => Promise.all([cleanUpEsResources()])); describe('clear cache', () => { diff --git a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/delete.ts b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/delete.ts index 415cdf4814176c..ed4bc8f4f8c7b7 100644 --- a/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/delete.ts +++ b/x-pack/test/saved_object_tagging/api_integration/tagging_api/apis/delete.ts @@ -13,7 +13,8 @@ export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); const supertest = getService('supertest'); - describe('DELETE /api/saved_objects_tagging/tags/{id}', () => { + // FLAKY: https://github.com/elastic/kibana/issues/90552 + describe.skip('DELETE /api/saved_objects_tagging/tags/{id}', () => { beforeEach(async () => { await esArchiver.load('delete_with_references'); });