Add response time and endpoint counters as metrics for Mira

src/Config.js

@@ -6,6 +6,7 @@ const defaultEngineMetricsPort = 9090;
 const defaultEngineDiscoveryInterval = 10000;
 const defaultEngineUpdateInterval = 10000;
 const defaultKubernetesProxyPort = 8001;
+const defaultAllowedResponseTime = 1000;
 const defaultDiscoveryLabel = 'qix-engine';
 const defaultEngineAPIPortLabel = 'qix-engine-api-port';
 const defaultEngineMetricsPortLabel = 'qix-engine-metrics-port';
@@ -149,6 +150,17 @@ class Config {
       Config.rollbarLevels = (process.env.MIRA_ROLLBAR_LEVELS || 'error').split(',');
       logger.info(`Mira will report '${Config.rollbarLevels}' log levels to Rollbar`);
     }
+
+    /**
+     * @prop {number} allowedResponseTime - The maximum allowed time in milliseconds from when a request is received by Mira
+     * until a response is being sent.
+     * @static
+     */
+    Config.allowedResponseTime = parseInt(process.env.MIRA_ALLOWED_RESPONSE_TIME, 10);
+    if (!Config.allowedResponseTime || isNaN(Config.allowedResponseTime)) {
+      Config.allowedResponseTime = defaultAllowedResponseTime;
+    }
+    logger.info(`Maximum allowed response time for Mira is: ${Config.allowedResponseTime} ms`);
   }
 }
 

src/Metrics.js

@@ -0,0 +1,45 @@
+const prom = require('prom-client');
+const Config = require('./Config');
+const logger = require('./logger/Logger').get();
+const version = require('../version');
+const os = require('os');
+
+// Create metric gauge containing build info from version.json
+new prom.Gauge({
+  name: `${version.name}_build_info`,
+  help: `A metric with a constant 1 value labeled by version, revision, platform, nodeVersion, os from which ${version.name} was built`,
+  labelNames: ['version', 'revision', 'buildTime', 'platform', 'nodeVersion', 'os', 'osRelease'],
+}).set({
+  version: version.version,
+  revision: version.SHA,
+  buildTime: version.buildTime,
+  platform: process.release.name,
+  nodeVersion: process.version,
+  os: process.platform,
+  osRelease: os.release(),
+}, 1);
+
+// Collect default prometheus metrics every 10 seconds
+const collectDefaultMetrics = prom.collectDefaultMetrics;
+collectDefaultMetrics();
+
+// Create metric summary for api response times
+const responseTimeSummary = new prom.Summary({
+  name: `${version.name}_api_response_time_ms`,
+  help: `Time in milliseconds consumed from ${version.name} receiving a request until a response is sent`,
+});
+
+// function for recording time consumed for a request and adding as metric
+function recordResponseTimes() {
+  return async function responseTime(ctx, next) {
+    const requestTime = Date.now();
+    await next();
+    const diff = Math.ceil(Date.now() - requestTime);
+    responseTimeSummary.observe(diff);
+    if (diff > Config.allowedResponseTime) {
+      logger.warn(`Request for endpoint ${ctx.request.url} took ${diff} ms, which is longer than allowed ${Config.allowedResponseTime} ms`);
+    }
+  };
+}
+
+module.exports = recordResponseTimes;

src/Routes.js

@@ -4,8 +4,7 @@ const Config = require('./Config');
 const Router = require('koa-router');
 const prom = require('prom-client');
 const logger = require('./logger/Logger').get();
-const version = require('../version');
-const os = require('os');
+
 
 const apiVersion = 'v1';
 const router = new Router({
@@ -18,29 +17,15 @@ const engineDiscovery = new EngineDiscovery(
   Config.engineDiscoveryInterval,
   Config.engineUpdateInterval);
 
-// Create metric gauge containing build info from version.json
-new prom.Gauge({
-  name: `${version.name}_build_info`,
-  help: `A metric with a constant 1 value labeled by version, revision, platform, nodeVersion, os from which ${version.name} was built`,
-  labelNames: ['version', 'revision', 'buildTime', 'platform', 'nodeVersion', 'os', 'osRelease'],
-}).set({
-  version: version.version,
-  revision: version.SHA,
-  buildTime: version.buildTime,
-  platform: process.release.name,
-  nodeVersion: process.version,
-  os: process.platform,
-  osRelease: os.release(),
-}, 1);
-
-// Collect default prometheus metrics every 10 seconds
-const collectDefaultMetrics = prom.collectDefaultMetrics;
-collectDefaultMetrics();
-
 const healthEndpoint = 'health';
 const metricsEndpoint = 'metrics';
 const enginesEndpoint = 'engines';
 
+// Initialize endpoint counters
+const enginesCounter = new prom.Counter({ name: 'mira_api_engines_request_counter', help: 'Number of requests to /engines endpoint' });
+const healthCounter = new prom.Counter({ name: 'mira_api_health_request_counter', help: 'Number of requests to /health endpoint' });
+const metricsCounter = new prom.Counter({ name: 'mira_api_metrics_request_counter', help: 'Number of requests to /metrics endpoint' });
+
 /**
  * @swagger
  * /health:
@@ -56,6 +41,7 @@ const enginesEndpoint = 'engines';
  */
 router.get(`/${healthEndpoint}`, async (ctx) => {
   logger.debug(`GET /${apiVersion}/${healthEndpoint}`);
+  healthCounter.inc();
   ctx.body = {};
 });
 
@@ -76,6 +62,7 @@ router.get(`/${healthEndpoint}`, async (ctx) => {
  */
 router.get(`/${metricsEndpoint}`, async (ctx) => {
   logger.debug(`GET /${apiVersion}/${metricsEndpoint}`);
+  metricsCounter.inc();
   if (ctx.accepts('text')) {
     ctx.body = prom.register.metrics();
   } else {
@@ -109,6 +96,7 @@ router.get(`/${metricsEndpoint}`, async (ctx) => {
  */
 router.get(`/${enginesEndpoint}`, async (ctx) => {
   logger.info(`GET /${apiVersion}/${enginesEndpoint}${ctx.querystring ? `?${ctx.querystring}` : ''}`);
+  enginesCounter.inc();
   try {
     ctx.body = await engineDiscovery.list(ctx.query);
   } catch (err) {

src/index.js

@@ -9,6 +9,7 @@ const Config = require('./Config');
 
 Config.init();
 
+const metrics = require('./Metrics');
 const router = require('./Routes');
 
 logger.info(`Build info: ${JSON.stringify(version)}`);
@@ -57,6 +58,7 @@ process.on('uncaughtException', onUnhandledError);
 process.on('unhandledRejection', onUnhandledError);
 
 app
+  .use(metrics())
   .use(swagger2koa.ui(document, '/openapi'))
   .use(router.routes())
   .use(router.allowedMethods());

test/integration/integration-local.spec.js

@@ -63,6 +63,10 @@ describe('GET /metrics', () => {
     expect(res.type).to.equal('text/plain');
     expect(res.text.length).to.not.equal(0);
     expect(res.text).to.contain('mira_build_info');
+    expect(res.text).to.contain('mira_api_response_time_ms');
+    expect(res.text).to.contain('mira_api_engines_request_counter');
+    expect(res.text).to.contain('mira_api_health_request_counter');
+    expect(res.text).to.contain('mira_api_metrics_request_counter');
   });
 });