[Telemetry] Separate the license retrieval from the stats in the usag…

…e collectors (#57332)

* [Telemetry] Merge OSS and XPack usage collectors

* Create X-Pack collector again

* Separate the license retrieval from the stats

* Fix telemetry tests with new fields

* Use Promise.all to retrieve license and stats at the same time

* Fix moment mock

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/legacy/core_plugins/telemetry/server/collection_manager.ts

@@ -20,6 +20,7 @@
 import { encryptTelemetry } from './collectors';
 import { CallCluster } from '../../elasticsearch';
 import { UsageCollectionSetup } from '../../../../plugins/usage_collection/server';
+import { ESLicense } from './telemetry_collection/get_local_license';
 
 export type EncryptedStatsGetterConfig = { unencrypted: false } & {
   server: any;
@@ -45,22 +46,38 @@ export interface StatsCollectionConfig {
   end: string | number;
 }
 
+export interface BasicStatsPayload {
+  timestamp: string;
+  cluster_uuid: string;
+  cluster_name: string;
+  version: string;
+  cluster_stats: object;
+  collection?: string;
+  stack_stats: object;
+}
+
 export type StatsGetterConfig = UnencryptedStatsGetterConfig | EncryptedStatsGetterConfig;
 export type ClusterDetailsGetter = (config: StatsCollectionConfig) => Promise<ClusterDetails[]>;
-export type StatsGetter = (
+export type StatsGetter<T extends BasicStatsPayload = BasicStatsPayload> = (
+  clustersDetails: ClusterDetails[],
+  config: StatsCollectionConfig
+) => Promise<T[]>;
+export type LicenseGetter = (
   clustersDetails: ClusterDetails[],
   config: StatsCollectionConfig
-) => Promise<any[]>;
+) => Promise<{ [clusterUuid: string]: ESLicense | undefined }>;
 
-interface CollectionConfig {
+interface CollectionConfig<T extends BasicStatsPayload> {
   title: string;
   priority: number;
   esCluster: string;
-  statsGetter: StatsGetter;
+  statsGetter: StatsGetter<T>;
   clusterDetailsGetter: ClusterDetailsGetter;
+  licenseGetter: LicenseGetter;
 }
 interface Collection {
   statsGetter: StatsGetter;
+  licenseGetter: LicenseGetter;
   clusterDetailsGetter: ClusterDetailsGetter;
   esCluster: string;
   title: string;
@@ -70,8 +87,15 @@ export class TelemetryCollectionManager {
   private usageGetterMethodPriority = -1;
   private collections: Collection[] = [];
 
-  public setCollection = (collectionConfig: CollectionConfig) => {
-    const { title, priority, esCluster, statsGetter, clusterDetailsGetter } = collectionConfig;
+  public setCollection = <T extends BasicStatsPayload>(collectionConfig: CollectionConfig<T>) => {
+    const {
+      title,
+      priority,
+      esCluster,
+      statsGetter,
+      clusterDetailsGetter,
+      licenseGetter,
+    } = collectionConfig;
 
     if (typeof priority !== 'number') {
       throw new Error('priority must be set.');
@@ -88,10 +112,14 @@ export class TelemetryCollectionManager {
         throw Error('esCluster name must be set for the getCluster method.');
       }
       if (!clusterDetailsGetter) {
-        throw Error('Cluser UUIds method is not set.');
+        throw Error('Cluster UUIds method is not set.');
+      }
+      if (!licenseGetter) {
+        throw Error('License getter method not set.');
       }
 
       this.collections.unshift({
+        licenseGetter,
         statsGetter,
         clusterDetailsGetter,
         esCluster,
@@ -141,7 +169,19 @@ export class TelemetryCollectionManager {
       return;
     }
 
-    return await collection.statsGetter(clustersDetails, statsCollectionConfig);
+    const [stats, licenses] = await Promise.all([
+      collection.statsGetter(clustersDetails, statsCollectionConfig),
+      collection.licenseGetter(clustersDetails, statsCollectionConfig),
+    ]);
+
+    return stats.map(stat => {
+      const license = licenses[stat.cluster_uuid];
+      return {
+        ...(license ? { license } : {}),
+        ...stat,
+        collectionSource: collection.title,
+      };
+    });
   };
 
   public getOptInStats = async (optInStatus: boolean, config: StatsGetterConfig) => {

src/legacy/core_plugins/telemetry/server/telemetry_collection/get_cluster_info.js → src/legacy/core_plugins/telemetry/server/telemetry_collection/get_cluster_info.ts

@@ -17,14 +17,32 @@
  * under the License.
  */
 
+import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
+
+// This can be removed when the ES client improves the types
+export interface ESClusterInfo {
+  cluster_uuid: string;
+  cluster_name: string;
+  version: {
+    number: string;
+    build_flavor: string;
+    build_type: string;
+    build_hash: string;
+    build_date: string;
+    build_snapshot?: boolean;
+    lucene_version: string;
+    minimum_wire_compatibility_version: string;
+    minimum_index_compatibility_version: string;
+  };
+}
+
 /**
  * Get the cluster info from the connected cluster.
  *
  * This is the equivalent to GET /
  *
  * @param {function} callCluster The callWithInternalUser handler (exposed for testing)
- * @return {Promise} The response from Elasticsearch.
  */
-export function getClusterInfo(callCluster) {
-  return callCluster('info');
+export function getClusterInfo(callCluster: CallCluster) {
+  return callCluster<ESClusterInfo>('info');
 }

src/legacy/core_plugins/telemetry/server/telemetry_collection/get_local_license.ts

@@ -0,0 +1,90 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { CallCluster } from 'src/legacy/core_plugins/elasticsearch';
+import { LicenseGetter } from '../collection_manager';
+
+// From https://www.elastic.co/guide/en/elasticsearch/reference/current/get-license.html
+export interface ESLicense {
+  status: string;
+  uid: string;
+  type: string;
+  issue_date: string;
+  issue_date_in_millis: number;
+  expiry_date: string;
+  expirty_date_in_millis: number;
+  max_nodes: number;
+  issued_to: string;
+  issuer: string;
+  start_date_in_millis: number;
+}
+let cachedLicense: ESLicense | undefined;
+
+function fetchLicense(callCluster: CallCluster, local: boolean) {
+  return callCluster<{ license: ESLicense }>('transport.request', {
+    method: 'GET',
+    path: '/_license',
+    query: {
+      local,
+      // For versions >= 7.6 and < 8.0, this flag is needed otherwise 'platinum' is returned for 'enterprise' license.
+      accept_enterprise: 'true',
+    },
+  });
+}
+
+/**
+ * Get the cluster's license from the connected node.
+ *
+ * This is the equivalent of GET /_license?local=true .
+ *
+ * Like any X-Pack related API, X-Pack must installed for this to work.
+ */
+async function getLicenseFromLocalOrMaster(callCluster: CallCluster) {
+  // Fetching the local license is cheaper than getting it from the master and good enough
+  const { license } = await fetchLicense(callCluster, true).catch(async err => {
+    if (cachedLicense) {
+      try {
+        // Fallback to the master node's license info
+        const response = await fetchLicense(callCluster, false);
+        return response;
+      } catch (masterError) {
+        if (masterError.statusCode === 404) {
+          // If the master node does not have a license, we can assume there is no license
+          cachedLicense = undefined;
+        } else {
+          // Any other errors from the master node, throw and do not send any telemetry
+          throw err;
+        }
+      }
+    }
+    return { license: void 0 };
+  });
+
+  if (license) {
+    cachedLicense = license;
+  }
+  return license;
+}
+
+export const getLocalLicense: LicenseGetter = async (clustersDetails, { callCluster }) => {
+  const license = await getLicenseFromLocalOrMaster(callCluster);
+
+  // It should be called only with 1 cluster element in the clustersDetails array, but doing reduce just in case.
+  return clustersDetails.reduce((acc, { clusterUuid }) => ({ ...acc, [clusterUuid]: license }), {});
+};

src/legacy/core_plugins/telemetry/server/telemetry_collection/get_local_stats.ts

@@ -17,11 +17,8 @@
  * under the License.
  */
 
-import { get, omit } from 'lodash';
-// @ts-ignore
-import { getClusterInfo } from './get_cluster_info';
+import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
 import { getClusterStats } from './get_cluster_stats';
-// @ts-ignore
 import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
 import { StatsGetter } from '../collection_manager';
 
@@ -33,35 +30,32 @@ import { StatsGetter } from '../collection_manager';
  * @param {Object} clusterInfo Cluster info (GET /)
  * @param {Object} clusterStats Cluster stats (GET /_cluster/stats)
  * @param {Object} kibana The Kibana Usage stats
- * @return {Object} A combined object containing the different responses.
  */
 export function handleLocalStats(
   server: any,
-  clusterInfo: any,
-  clusterStats: any,
+  { cluster_name, cluster_uuid, version }: ESClusterInfo,
+  { _nodes, cluster_name: clusterName, ...clusterStats }: any,
   kibana: KibanaUsageStats
 ) {
   return {
     timestamp: new Date().toISOString(),
-    cluster_uuid: get(clusterInfo, 'cluster_uuid'),
-    cluster_name: get(clusterInfo, 'cluster_name'),
-    version: get(clusterInfo, 'version.number'),
-    cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
+    cluster_uuid,
+    cluster_name,
+    version: version.number,
+    cluster_stats: clusterStats,
     collection: 'local',
     stack_stats: {
       kibana: handleKibanaStats(server, kibana),
     },
   };
 }
 
+export type TelemetryLocalStats = ReturnType<typeof handleLocalStats>;
+
 /**
  * Get statistics for all products joined by Elasticsearch cluster.
- *
- * @param {Object} server The Kibana server instance used to call ES as the internal user
- * @param {function} callCluster The callWithInternalUser handler (exposed for testing)
- * @return {Promise} The object containing the current Elasticsearch cluster's telemetry.
  */
-export const getLocalStats: StatsGetter = async (clustersDetails, config) => {
+export const getLocalStats: StatsGetter<TelemetryLocalStats> = async (clustersDetails, config) => {
   const { server, callCluster, usageCollection } = config;
 
   return await Promise.all(

src/legacy/core_plugins/telemetry/server/telemetry_collection/index.ts

@@ -17,7 +17,6 @@
  * under the License.
  */
 
-// @ts-ignore
 export { getLocalStats } from './get_local_stats';
 export { getClusterUuids } from './get_cluster_stats';
 export { registerCollection } from './register_collection';

src/legacy/core_plugins/telemetry/server/telemetry_collection/register_collection.ts

@@ -39,6 +39,7 @@
 import { telemetryCollectionManager } from '../collection_manager';
 import { getLocalStats } from './get_local_stats';
 import { getClusterUuids } from './get_cluster_stats';
+import { getLocalLicense } from './get_local_license';
 
 export function registerCollection() {
   telemetryCollectionManager.setCollection({
@@ -47,5 +48,6 @@ export function registerCollection() {
     priority: 0,
     statsGetter: getLocalStats,
     clusterDetailsGetter: getClusterUuids,
+    licenseGetter: getLocalLicense,
   });
 }

src/plugins/telemetry/public/services/telemetry_service.test.ts

@@ -26,9 +26,18 @@ const mockSubtract = jest.fn().mockImplementation(() => {
   };
 });
 
+const mockClone = jest.fn().mockImplementation(() => {
+  return {
+    clone: mockClone,
+    subtract: mockSubtract,
+    toISOString: jest.fn(),
+  };
+});
+
 jest.mock('moment', () => {
   return jest.fn().mockImplementation(() => {
     return {
+      clone: mockClone,
       subtract: mockSubtract,
       toISOString: jest.fn(),
     };
@@ -43,6 +52,7 @@ describe('TelemetryService', () => {
       expect(telemetryService['http'].post).toBeCalledWith('/api/telemetry/v2/clusters/_stats', {
         body: JSON.stringify({ unencrypted: false, timeRange: {} }),
       });
+      expect(mockClone).toBeCalled();
       expect(mockSubtract).toBeCalledWith(20, 'minutes');
     });
   });

src/plugins/telemetry/public/services/telemetry_service.ts

@@ -92,7 +92,10 @@ export class TelemetryService {
       body: JSON.stringify({
         unencrypted,
         timeRange: {
-          min: now.subtract(20, 'minutes').toISOString(),
+          min: now
+            .clone() // Need to clone it to avoid mutation (and max being the same value)
+            .subtract(20, 'minutes')
+            .toISOString(),
           max: now.toISOString(),
         },
       }),

x-pack/legacy/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts

@@ -5,7 +5,7 @@
  */
 
 import sinon from 'sinon';
-import { addStackStats, getAllStats, handleAllStats } from './get_all_stats';
+import { getStackStats, getAllStats, handleAllStats } from './get_all_stats';
 import { ESClusterStats } from './get_es_stats';
 import { KibanaStats } from './get_kibana_stats';
 import { ClustersHighLevelStats } from './get_high_level_stats';
@@ -223,7 +223,8 @@ describe('get_all_stats', () => {
         beats: {},
       });
 
-      expect(clusters).toStrictEqual(expectedClusters);
+      const [a, b, c] = expectedClusters;
+      expect(clusters).toStrictEqual([a, b, { ...c, stack_stats: {} }]);
     });
 
     it('handles no clusters response', () => {
@@ -233,9 +234,8 @@ describe('get_all_stats', () => {
     });
   });
 
-  describe('addStackStats', () => {
+  describe('getStackStats', () => {
     it('searches for clusters', () => {
-      const cluster = { cluster_uuid: 'a' };
       const stats = {
         a: {
           count: 2,
@@ -250,9 +250,7 @@ describe('get_all_stats', () => {
         },
       };
 
-      addStackStats(cluster as ESClusterStats, stats, 'xyz');
-
-      expect((cluster as any).stack_stats.xyz).toStrictEqual(stats.a);
+      expect(getStackStats('a', stats, 'xyz')).toStrictEqual({ xyz: stats.a });
     });
   });
 });