[Stats API] Set API field names per spec

src/server/kbn_server.js

@@ -27,8 +27,8 @@ import configSetupMixin from './config/setup';
 import httpMixin from './http';
 import { loggingMixin } from './logging';
 import warningsMixin from './warnings';
-import { statusMixin } from './status';
 import { usageMixin } from './usage';
+import { statusMixin } from './status';
 import pidMixin from './pid';
 import { configDeprecationWarningsMixin } from './config/deprecation_warnings';
 import configCompleteMixin from './config/complete';
@@ -64,8 +64,8 @@ export default class KbnServer {
       loggingMixin,
       configDeprecationWarningsMixin,
       warningsMixin,
-      statusMixin,
       usageMixin,
+      statusMixin,
 
       // writes pid file
       pidMixin,

src/server/status/index.js

@@ -32,11 +32,8 @@ export function statusMixin(kbnServer, server, config) {
     const metrics = new Metrics(config, server);
 
     evenBetter.monitor.on('ops', event => {
-      // for status API (to deprecate in next major)
-      metrics.capture(event).then(data => { kbnServer.metrics = data; });
-
-      // for metrics API (replacement API)
-      collector.collect(event); // collect() is async, but here we aren't depending on the return value
+      metrics.capture(event).then(data => { kbnServer.metrics = data; }); // for status API (to deprecate in next major)
+      collector.collect(event); // for metrics API (replacement API)
     });
   }
 

src/server/status/lib/index.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { setApiFieldNames } from './set_api_field_names';

src/server/status/lib/set_api_field_names.js

@@ -0,0 +1,41 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+function getValueOrRecurse(value) {
+  if (value == null || typeof value !== 'object') {
+    return value;
+  } else {
+    return setApiFieldNames(value); // recurse
+  }
+}
+
+export function setApiFieldNames(apiData) {
+  return Object.keys(apiData).reduce((accum, currName) => {
+    const value = apiData[currName];
+
+    let newName = currName;
+    newName = newName.replace('_in_bytes', '_bytes');
+    newName = newName.replace('_in_millis', '_ms');
+
+    return {
+      ...accum,
+      [newName]: getValueOrRecurse(value),
+    };
+  }, {});
+}

src/server/status/metrics_collector/metrics.js

@@ -56,8 +56,7 @@ export class Metrics {
 
     const metrics = {
       last_updated: timestamp,
-      collection_interval_in_millis: this.config.get('ops.interval'),
-      uptime_in_millis: event.process.uptime_ms, // TODO: deprecate this field, data should only have process.uptime_ms
+      collection_interval_in_millis: this.config.get('ops.interval')
     };
 
     return merge(metrics, event, cgroup);
@@ -92,10 +91,10 @@ export class Metrics {
         mem: {
           free_in_bytes: os.freemem(),
           total_in_bytes: os.totalmem()
-        }
+        },
+        uptime_ms: os.uptime() * 1000
       },
       response_times: {
-        // TODO: rename to use `_ms` suffix per beats naming conventions
         avg_in_millis: isNaN(avgInMillis) ? undefined : avgInMillis, // convert NaN to undefined
         max_in_millis: maxInMillis
       },

src/server/status/metrics_collector/metrics_collector.js

@@ -29,17 +29,18 @@ const matchSnapshot = /-SNAPSHOT$/;
 export class MetricsCollector {
   constructor(server, config) {
 
-    // NOTE we need access to config every time this is used because uuid is managed by the kibana core_plugin, which is initialized AFTER kbn_server
-    this._getBaseStats = () => ({
-      name: config.get('server.name'),
-      uuid: config.get('server.uuid'),
-      version: {
-        number: config.get('pkg.version').replace(matchSnapshot, ''),
-        build_hash: config.get('pkg.buildSha'),
-        build_number: config.get('pkg.buildNum'),
-        build_snapshot: matchSnapshot.test(config.get('pkg.version'))
-      }
-    });
+    // NOTE we need access to config every time this is used because uuid is
+    // managed by the kibana core_plugin, which is initialized AFTER kbn_server
+    this._getKibanaStats = () => {
+      return {
+        name: config.get('server.name'),
+        host: config.get('server.host'),
+        uuid: config.get('server.uuid'),
+        transport_address: `${config.get('server.host')}:${config.get('server.port')}`,
+        version: config.get('pkg.version'),
+        snapshot: matchSnapshot.test(config.get('pkg.version')),
+      };
+    };
 
     this._stats = Metrics.getStubMetrics();
     this._metrics = new Metrics(config, server); // TODO: deprecate status API that uses Metrics class, move it this module, fix the order of its constructor params
@@ -104,9 +105,13 @@ export class MetricsCollector {
     return stats;
   }
 
-  getStats() {
+  getStats(kbnServer) {
+    const status = kbnServer.status.toJSON();
     return {
-      ...this._getBaseStats(),
+      kibana: {
+        ...this._getKibanaStats(),
+        status: status.overall.state
+      },
       ...this._stats
     };
   }

src/server/status/routes/api/register_stats.js

@@ -19,10 +19,35 @@
 
 import Joi from 'joi';
 import { wrapAuthConfig } from '../../wrap_auth_config';
+import { setApiFieldNames } from '../../lib';
+
+async function getExtended(req, server) {
+  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('admin'); // admin cluster, get info on internal system
+  const callCluster = (...args) => callWithRequest(req, ...args);
+
+  let clusterUuid;
+  try {
+    const { cluster_uuid: uuid } = await callCluster('info', { filterPath: 'cluster_uuid', });
+    clusterUuid = uuid;
+  } catch (err) {
+    clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
+  }
+
+  let usage;
+  try {
+    const { collectorSet } = server.usage;
+    const usageRaw = await collectorSet.bulkFetchUsage(callCluster);
+    usage = collectorSet.summarizeStats(usageRaw);
+  } catch (err) {
+    usage = undefined;
+  }
+
+  return { clusterUuid, usage };
+}
 
 /*
  * API for Kibana meta info and accumulated operations stats
- * Including ?extended in the query string fetches Elasticsearch cluster_uuid
+ * Including ?extended in the query string fetches Elasticsearch cluster_uuid and server.usage.collectorSet data
  * - Requests to set isExtended = true
  *      GET /api/stats?extended=true
  *      GET /api/stats?extended
@@ -48,22 +73,16 @@ export function registerStatsApi(kbnServer, server, config, collector) {
         const isExtended = extended !== undefined && extended !== 'false';
 
         let clusterUuid;
+        let usage;
         if (isExtended) {
-          try {
-            const { callWithRequest, } = server.plugins.elasticsearch.getCluster('data');
-            const { cluster_uuid: uuid } = await callWithRequest(req, 'info', { filterPath: 'cluster_uuid', });
-            clusterUuid = uuid;
-          } catch (err) {
-            clusterUuid = undefined; // fallback from anonymous access or auth failure, redundant for explicitness
-          }
+          ({ clusterUuid, usage } = await getExtended(req, server));
         }
 
-        const stats = {
-          cluster_uuid: clusterUuid, // serialization makes an undefined get stripped out, as undefined isn't a JSON type
-          status: kbnServer.status.toJSON(),
-          ...collector.getStats(),
-        };
-
+        const stats = setApiFieldNames({
+          ...collector.getStats(kbnServer),
+          cluster_uuid: clusterUuid,
+          usage,
+        });
         reply(stats);
       },
     })

src/server/status/routes/api/register_status.js

@@ -41,7 +41,10 @@ export function registerStatusApi(kbnServer, server, config) {
           build_snapshot: matchSnapshot.test(config.get('pkg.version'))
         },
         status: kbnServer.status.toJSON(),
-        metrics: kbnServer.metrics
+        metrics: {
+          ...kbnServer.metrics,
+          uptime_in_millis: process.uptime() * 1000
+        }
       };
 
       return reply(status);

src/server/usage/classes/collector_set.js

@@ -89,14 +89,16 @@ export class CollectorSet {
 
   async bulkFetchUsage(callCluster) {
     const usageCollectors = this._collectors.filter(c => c instanceof UsageCollector);
-    const bulk = await this.bulkFetch(callCluster, usageCollectors);
+    return this.bulkFetch(callCluster, usageCollectors);
 
-    // summarize each type of stat
-    return bulk.reduce((accumulatedStats, currentStat) => {
-      /* Suffix removal is a temporary hack: some types have `_stats` suffix
-       * because of how monitoring bulk upload needed to combine types. It can
-       * be removed when bulk upload goes away
-       */
+  }
+
+  /*
+   * Summarize the data returned by bulk fetching into a simpler format
+   */
+  summarizeStats(statsData) {
+    return statsData.reduce((accumulatedStats, currentStat) => {
+      // `_stats` Suffix removal
       const statType = currentStat.type.replace('_stats', '');
       return {
         ...accumulatedStats,

x-pack/plugins/monitoring/server/kibana_monitoring/init.js

@@ -10,7 +10,7 @@ import { getCollectorTypesCombiner } from './lib';
 /**
  * Initialize different types of Kibana Monitoring
  * TODO: remove this in 7.0
- * - Ops Events - essentially Kibana's /api/status
+ * - Ops Events - essentially Kibana's /api/status (non-rolled up)
  * - Usage Stats - essentially Kibana's /api/stats
  * - Kibana Settings - select uiSettings
  * @param {Object} kbnServer manager of Kibana services - see `src/server/kbn_server` in Kibana core
@@ -25,4 +25,3 @@ export function initBulkUploader(kbnServer, server) {
     combineTypes: getCollectorTypesCombiner(kbnServer, config)
   });
 }
-

x-pack/plugins/xpack_main/server/routes/api/v1/xpack_usage.js

@@ -15,9 +15,10 @@ const getClusterUuid = async callCluster => {
  * @return {Object} data from usage stats collectors registered with Monitoring CollectorSet
  * @throws {Error} if the Monitoring CollectorSet is not ready
  */
-const getUsage = (callCluster, server) => {
+const getUsage = async (callCluster, server) => {
   const { collectorSet } = server.usage;
-  return collectorSet.bulkFetchUsage(callCluster);
+  const usage = await collectorSet.bulkFetchUsage(callCluster);
+  return collectorSet.summarizeStats(usage);
 };
 
 export function xpackUsageRoute(server) {