[FLEET][SECURITY_SOLUTION] Fix .fleet-artifacs index property names and add endpoint package policy migrations

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -38,8 +38,7 @@ import {
   migrateAgentToV7120,
   migratePackagePolicyToV7120,
 } from './migrations/to_v7_12_0';
-
-import { migrateSettingsToV7130 } from './migrations/to_v7_13_0';
+import { migratePackagePolicyToV7130, migrateSettingsToV7130 } from './migrations/to_v7_13_0';
 
 /*
  * Saved object types and mappings
@@ -289,6 +288,7 @@ const getSavedObjectTypes = (
       '7.10.0': migratePackagePolicyToV7100,
       '7.11.0': migratePackagePolicyToV7110,
       '7.12.0': migratePackagePolicyToV7120,
+      '7.13.0': migratePackagePolicyToV7130,
     },
   },
   [PACKAGES_SAVED_OBJECT_TYPE]: {

x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts

@@ -7,3 +7,4 @@
 
 export { migratePackagePolicyToV7110 } from './to_v7_11_0';
 export { migratePackagePolicyToV7120 } from './to_v7_12_0';
+export { migrateEndpointPackagePolicyToV7130 } from './to_v7_13_0';

x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_13_0.test.ts

@@ -0,0 +1,145 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectUnsanitizedDoc } from 'kibana/server';
+import { cloneDeepWith, cloneDeep } from 'lodash';
+
+import type { PackagePolicy } from '../../../../common';
+
+import { migrationMocks } from '../../../../../../../src/core/server/mocks';
+
+import { migrateEndpointPackagePolicyToV7130 } from './to_v7_13_0';
+
+describe('7.13.0 Endpoint Package Policy migration', () => {
+  const createOldPackagePolicySO = (): SavedObjectUnsanitizedDoc<PackagePolicy> => {
+    return {
+      id: 'mock-saved-object-id',
+      attributes: {
+        name: 'Some Policy Name',
+        package: {
+          name: 'endpoint',
+          title: '',
+          version: '',
+        },
+        id: 'mock-saved-object-id',
+        policy_id: '',
+        enabled: true,
+        namespace: '',
+        output_id: '',
+        revision: 0,
+        updated_at: '',
+        updated_by: '',
+        created_at: '',
+        created_by: '',
+        inputs: [
+          {
+            type: 'endpoint',
+            enabled: true,
+            streams: [],
+            config: {
+              artifact_manifest: {
+                value: {
+                  manifest_version: '1.0.0',
+                  schema_version: 'v1',
+                  artifacts: {
+                    'endpoint-exceptionlist-macos-v1': {
+                      encryption_algorithm: 'none',
+                      decoded_sha256:
+                        'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      decoded_size: 14,
+                      encoded_sha256:
+                        'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+                      encoded_size: 22,
+                      relative_url:
+                        '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      compression_algorithm: 'zlib',
+                    },
+                    'endpoint-exceptionlist-windows-v1': {
+                      encryption_algorithm: 'none',
+                      decoded_sha256:
+                        'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      decoded_size: 14,
+                      encoded_sha256:
+                        'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+                      encoded_size: 22,
+                      relative_url:
+                        '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      compression_algorithm: 'zlib',
+                    },
+                    'endpoint-trustlist-macos-v1': {
+                      encryption_algorithm: 'none',
+                      decoded_sha256:
+                        'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      decoded_size: 14,
+                      encoded_sha256:
+                        'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+                      encoded_size: 22,
+                      relative_url:
+                        '/api/endpoint/artifacts/download/endpoint-trustlist-macos-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      compression_algorithm: 'zlib',
+                    },
+                    'endpoint-trustlist-windows-v1': {
+                      encryption_algorithm: 'none',
+                      decoded_sha256:
+                        'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      decoded_size: 14,
+                      encoded_sha256:
+                        'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+                      encoded_size: 22,
+                      relative_url:
+                        '/api/endpoint/artifacts/download/endpoint-trustlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      compression_algorithm: 'zlib',
+                    },
+                    'endpoint-trustlist-linux-v1': {
+                      encryption_algorithm: 'none',
+                      decoded_sha256:
+                        'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      decoded_size: 14,
+                      encoded_sha256:
+                        'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
+                      encoded_size: 22,
+                      relative_url:
+                        '/api/endpoint/artifacts/download/endpoint-trustlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
+                      compression_algorithm: 'zlib',
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
+      },
+      type: ' nested',
+    };
+  };
+  const createNewPackagePolicySO = (): SavedObjectUnsanitizedDoc<PackagePolicy> => {
+    return cloneDeepWith(createOldPackagePolicySO(), (value, key) => {
+      if (key === 'relative_url') {
+        return value.replace('/api/endpoint/artifacts/download/', '/api/fleet/artifacts/');
+      }
+    });
+  };
+
+  const migrationContext = migrationMocks.createContext();
+
+  it('should adjust the relative url for all artifact manifests', () => {
+    expect(
+      migrateEndpointPackagePolicyToV7130(createOldPackagePolicySO(), migrationContext)
+    ).toEqual(createNewPackagePolicySO());
+  });
+
+  it('should NOT touch non-endpoint package policies', () => {
+    const packagePolicySo = createOldPackagePolicySO();
+    packagePolicySo.attributes.package!.name = 'not endpoint';
+
+    const unchangedPackagePolicySo = cloneDeep(packagePolicySo);
+
+    expect(migrateEndpointPackagePolicyToV7130(packagePolicySo, migrationContext)).toEqual(
+      unchangedPackagePolicySo
+    );
+  });
+});

x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v7_13_0.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectMigrationFn } from 'kibana/server';
+
+import type { PackagePolicy } from '../../../../common';
+import { relativeDownloadUrlFromArtifact } from '../../../services/artifacts/mappings';
+import type { ArtifactElasticsearchProperties } from '../../../services';
+
+type ArtifactManifestList = Record<
+  string,
+  Pick<ArtifactElasticsearchProperties, 'identifier' | 'decoded_sha256' | 'relative_url'>
+>;
+
+export const migrateEndpointPackagePolicyToV7130: SavedObjectMigrationFn<
+  PackagePolicy,
+  PackagePolicy
+> = (packagePolicyDoc) => {
+  if (packagePolicyDoc.attributes.package?.name === 'endpoint') {
+    // Adjust all artifact URLs so that they point at fleet-server
+    const artifactList: ArtifactManifestList =
+      packagePolicyDoc.attributes?.inputs[0]?.config?.artifact_manifest.value.artifacts;
+
+    if (artifactList) {
+      for (const [identifier, artifactManifest] of Object.entries(artifactList)) {
+        artifactManifest.relative_url = relativeDownloadUrlFromArtifact({
+          identifier,
+          decodedSha256: artifactManifest.decoded_sha256,
+        });
+      }
+    }
+  }
+
+  return packagePolicyDoc;
+};

x-pack/plugins/fleet/server/saved_objects/migrations/to_v7_13_0.ts

@@ -9,6 +9,10 @@ import type { SavedObjectMigrationFn } from 'kibana/server';
 
 import type { Settings } from '../../types';
 
+import type { PackagePolicy } from '../../../common';
+
+import { migrateEndpointPackagePolicyToV7130 } from './security_solution';
+
 export const migrateSettingsToV7130: SavedObjectMigrationFn<
   Settings & {
     package_auto_upgrade: string;
@@ -23,3 +27,16 @@ export const migrateSettingsToV7130: SavedObjectMigrationFn<
 
   return settingsDoc;
 };
+
+export const migratePackagePolicyToV7130: SavedObjectMigrationFn<PackagePolicy, PackagePolicy> = (
+  packagePolicyDoc,
+  migrationContext
+) => {
+  // Endpoint specific migrations
+  // FIXME:PT remove `-OFF` from below once ready to be released
+  if (packagePolicyDoc.attributes.package?.name === 'endpoint-OFF') {
+    return migrateEndpointPackagePolicyToV7130(packagePolicyDoc, migrationContext);
+  }
+
+  return packagePolicyDoc;
+};

x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts

@@ -30,6 +30,7 @@ import {
 } from './artifacts';
 
 import type { NewArtifact } from './types';
+import { newArtifactToElasticsearchProperties } from './mappings';
 
 describe('When using the artifacts services', () => {
   let esClientMock: ReturnType<typeof elasticsearchServiceMock.createInternalClient>;
@@ -86,7 +87,7 @@ describe('When using the artifacts services', () => {
         index: FLEET_SERVER_ARTIFACTS_INDEX,
         id: expect.any(String),
         body: {
-          ...newArtifact,
+          ...newArtifactToElasticsearchProperties(newArtifact),
           created: expect.any(String),
         },
         refresh: 'wait_for',

x-pack/plugins/fleet/server/services/artifacts/artifacts.ts

@@ -29,7 +29,7 @@ import type {
   ListArtifactsProps,
   NewArtifact,
 } from './types';
-import { esSearchHitToArtifact } from './mappings';
+import { esSearchHitToArtifact, newArtifactToElasticsearchProperties } from './mappings';
 
 const deflateAsync = promisify(deflate);
 
@@ -58,10 +58,7 @@ export const createArtifact = async (
   artifact: NewArtifact
 ): Promise<Artifact> => {
   const id = uuid.v4();
-  const newArtifactData: ArtifactElasticsearchProperties = {
-    ...artifact,
-    created: new Date().toISOString(),
-  };
+  const newArtifactData = newArtifactToElasticsearchProperties(artifact);
 
   try {
     await esClient.create({
@@ -71,10 +68,7 @@ export const createArtifact = async (
       refresh: 'wait_for',
     });
 
-    return {
-      ...newArtifactData,
-      id,
-    };
+    return esSearchHitToArtifact({ _id: id, _source: newArtifactData });
   } catch (e) {
     throw new ArtifactsElasticsearchError(e);
   }

x-pack/plugins/fleet/server/services/artifacts/client.test.ts

@@ -25,7 +25,7 @@ describe('When using the Fleet Artifacts Client', () => {
     const singleHit = generateArtifactEsGetSingleHitMock();
 
     if (withInvalidArtifact) {
-      singleHit._source.packageName = 'not endpoint';
+      singleHit._source.package_name = 'not endpoint';
     }
 
     esClientMock.get.mockImplementation(() => {
@@ -129,7 +129,7 @@ describe('When using the Fleet Artifacts Client', () => {
 
       expect(esClientMock.search).toHaveBeenCalledWith(
         expect.objectContaining({
-          q: '(packageName: "endpoint") AND identifier: one',
+          q: '(package_name: "endpoint") AND identifier: one',
         })
       );
     });
@@ -143,7 +143,7 @@ describe('When using the Fleet Artifacts Client', () => {
       });
       expect(esClientMock.search).toHaveBeenCalledWith(
         expect.objectContaining({
-          q: '(packageName: "endpoint")',
+          q: '(package_name: "endpoint")',
         })
       );
     });

x-pack/plugins/fleet/server/services/artifacts/client.ts

@@ -85,12 +85,17 @@ export class FleetArtifactsClient implements ArtifactsClientInterface {
     }
   }
 
+  /**
+   * Get a list of artifacts.
+   * NOTE that when using the `kuery` filtering param, that all filters property names should
+   * match the internal attribute names of the index
+   */
   async listArtifacts({ kuery, ...options }: ListArtifactsProps = {}): Promise<
     ListResult<Artifact>
   > {
     // All filtering for artifacts should be bound to the `packageName`, so we insert
     // that into the KQL value and use `AND` to add the defined `kuery` (if any) to it.
-    const filter = `(packageName: "${this.packageName}")${kuery ? ` AND ${kuery}` : ''}`;
+    const filter = `(package_name: "${this.packageName}")${kuery ? ` AND ${kuery}` : ''}`;
 
     return listArtifacts(this.esClient, {
       ...options,

x-pack/plugins/fleet/server/services/artifacts/mappings.ts

@@ -7,17 +7,57 @@
 
 import type { ESSearchHit } from '../../../../../../typings/elasticsearch';
 
-import type { Artifact, ArtifactElasticsearchProperties } from './types';
+import type { Artifact, ArtifactElasticsearchProperties, NewArtifact } from './types';
 import { ARTIFACT_DOWNLOAD_RELATIVE_PATH } from './constants';
 
 export const esSearchHitToArtifact = <
   T extends Pick<ESSearchHit<ArtifactElasticsearchProperties>, '_id' | '_source'>
->(
-  searchHit: T
-): Artifact => {
+>({
+  _id: id,
+  _source: {
+    compression_algorithm: compressionAlgorithm,
+    decoded_sha256: decodedSha256,
+    decoded_size: decodedSize,
+    encoded_sha256: encodedSha256,
+    encoded_size: encodedSize,
+    encryption_algorithm: encryptionAlgorithm,
+    package_name: packageName,
+    ...attributesNotNeedingRename
+  },
+}: T): Artifact => {
+  return {
+    ...attributesNotNeedingRename,
+    id,
+    compressionAlgorithm,
+    decodedSha256,
+    decodedSize,
+    encodedSha256,
+    encodedSize,
+    encryptionAlgorithm,
+    packageName,
+  };
+};
+
+export const newArtifactToElasticsearchProperties = ({
+  encryptionAlgorithm,
+  packageName,
+  encodedSize,
+  encodedSha256,
+  decodedSize,
+  decodedSha256,
+  compressionAlgorithm,
+  ...attributesNotNeedingRename
+}: NewArtifact): ArtifactElasticsearchProperties => {
   return {
-    ...searchHit._source,
-    id: searchHit._id,
+    ...attributesNotNeedingRename,
+    encryption_algorithm: encryptionAlgorithm,
+    package_name: packageName,
+    encoded_size: encodedSize,
+    encoded_sha256: encodedSha256,
+    decoded_size: decodedSize,
+    decoded_sha256: decodedSha256,
+    compression_algorithm: compressionAlgorithm,
+    created: new Date().toISOString(),
   };
 };