feat: disableSend boolean config option to work, but not communicate …

…with APM server (elastic#2127) Env var is ELASTIC_APM_DISABLE_SEND. Default is false. This feature is useful to use the APM agent for async context tracking, distributed tracing context propagation, and/or log enrichment with trace.id. When disableSend=true, attempts are made to limit unnecessary processing (like calculating metrics, encoding stacktraces, etc.). This also includes an improvement to the `agent configured correctly` log.trace that dumps the agent config.
dgieselaar · Sep 10, 2021 · 76b3899 · 76b3899
1 parent a0350d3
commit 76b3899
Show file tree

Hide file tree

Showing 13 changed files with 337 additions and 10 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -38,6 +38,8 @@ Notes:
 
 * Add instrumentation of all AWS S3 methods when using the
   https://www.npmjs.com/package/aws-sdk[JavaScript AWS SDK v2] (`aws-sdk`).
+* Add <<disable-send, `disableSend`>> configuration option. This supports some
+  use cases using the APM agent **without** an APM server. ({issues}2101[#2101])
 
 [float]
 ===== Bug fixes

diff --git a/docs/configuration.asciidoc b/docs/configuration.asciidoc
@@ -148,6 +148,22 @@ See {kibana-ref}/filters.html#environment-selector[environment selector] in the
 NOTE: This feature is fully supported in the APM app in Kibana versions >= 7.2.
 You must use the query bar to filter for a specific environment in versions prior to 7.2.
 
+[[disable-send]]
+==== `disableSend`
+
+* *Type:* Boolean
+* *Default:* `false`
+* *Env:* `ELASTIC_APM_DISABLE_SEND`
+
+If set to `true`, the agent will work as usual, except for any task requiring
+communication with the APM server. Events will be dropped and the agent won't be
+able to receive central configuration, which means that any other configuration
+cannot be changed in this state without restarting the service.  Example uses
+for this setting are: maintaining the ability to create traces and log
+trace/transaction/span IDs through the log correlation feature, and getting
+automatic distributed tracing via the https://w3c.github.io/trace-context/[W3C
+HTTP headers].
+
 [[instrument]]
 ==== `instrument`
 

diff --git a/index.d.ts b/index.d.ts
@@ -207,6 +207,7 @@ interface AgentConfigOptions {
   configFile?: string;
   containerId?: string;
   disableInstrumentations?: string | string[];
+  disableSend?: boolean;
   environment?: string;
   errorMessageMaxLength?: string; // Also support `number`, but as we're removing this functionality soon, there's no need to advertise it
   errorOnAbortedRequests?: boolean;

diff --git a/lib/agent.js b/lib/agent.js
@@ -198,7 +198,7 @@ Agent.prototype.start = function (opts) {
       pkg = require(path.join(basedir, 'package.json'))
     } catch (e) {}
 
-    this.logger.trace('agent configured correctly %o', {
+    this.logger.trace({
       pid: process.pid,
       ppid: process.ppid,
       arch: process.arch,
@@ -209,7 +209,7 @@ Agent.prototype.start = function (opts) {
       main: pkg ? pkg.main : '<could not determine>',
       dependencies: pkg ? pkg.dependencies : '<could not determine>',
       conf: this._conf.toJSON()
-    })
+    }, 'agent configured correctly')
   }
 
   this._transport = this._conf.transport(this._conf, this)
@@ -361,6 +361,14 @@ Agent.prototype.captureError = function (err, opts, cb) {
     opts = EMPTY_OPTS
   }
 
+  // Quick out if disableSend=true, no point in the processing time.
+  if (this._conf.disableSend) {
+    if (cb) {
+      process.nextTick(cb, null, errors.generateErrorId())
+    }
+    return
+  }
+
   const agent = this
   let callSiteLoc = null
   const errIsError = isError(err)

diff --git a/lib/config.js b/lib/config.js
@@ -13,6 +13,7 @@ var packageName = require('../package').name
 
 const { WildcardMatcher } = require('./wildcard-matcher')
 const { CloudMetadata } = require('./cloud-metadata')
+const { NoopTransport } = require('./noop-transport')
 
 // Standardize user-agent header. Only use "elasticapm-node" if it matches "elastic-apm-node".
 if (packageName === 'elastic-apm-node') {
@@ -53,6 +54,7 @@ var DEFAULTS = {
   cloudProvider: 'auto',
   containerId: undefined,
   disableInstrumentations: [],
+  disableSend: false,
   environment: process.env.NODE_ENV || 'development',
   errorMessageMaxLength: '2kb',
   errorOnAbortedRequests: false,
@@ -110,6 +112,7 @@ var ENV_TABLE = {
   containerId: 'ELASTIC_APM_CONTAINER_ID',
   cloudProvider: 'ELASTIC_APM_CLOUD_PROVIDER',
   disableInstrumentations: 'ELASTIC_APM_DISABLE_INSTRUMENTATIONS',
+  disableSend: 'ELASTIC_APM_DISABLE_SEND',
   environment: 'ELASTIC_APM_ENVIRONMENT',
   ignoreMessageQueues: 'ELASTIC_IGNORE_MESSAGE_QUEUES',
   errorMessageMaxLength: 'ELASTIC_APM_ERROR_MESSAGE_MAX_LENGTH',
@@ -171,6 +174,7 @@ var BOOL_OPTS = [
   'captureHeaders',
   'captureSpanStackTraces',
   'centralConfig',
+  'disableSend',
   'errorOnAbortedRequests',
   'filterHttpHeaders',
   'instrument',
@@ -264,7 +268,11 @@ class Config {
 
     normalize(this, this.logger)
 
-    if (typeof this.transport !== 'function') {
+    if (this.disableSend) {
+      this.transport = function createNoopTransport (conf, agent) {
+        return new NoopTransport()
+      }
+    } else if (typeof this.transport !== 'function') {
       this.transport = function httpTransport (conf, agent) {
         let clientLogger = null
         if (!logging.isLoggerCustom(agent.logger)) {
@@ -399,9 +407,21 @@ class Config {
       logger: true,
       transport: true
     }
+    const NICE_REGEXPS_FIELDS = {
+      ignoreUrlRegExp: true,
+      ignoreUserAgentRegExp: true,
+      transactionIgnoreUrlRegExp: true,
+      sanitizeFieldNamesRegExp: true,
+      ignoreMessageQueuesRegExp: true
+    }
     const loggable = {}
     for (const k in this) {
-      if (!EXCLUDE_FIELDS[k] && this[k] !== undefined) {
+      if (EXCLUDE_FIELDS[k] || this[k] === undefined) {
+        // pass
+      } else if (NICE_REGEXPS_FIELDS[k] && Array.isArray(this[k])) {
+        // JSON.stringify() on a RegExp is "{}", which isn't very helpful.
+        loggable[k] = this[k].map(r => r instanceof RegExp ? r.toString() : r)
+      } else {
         loggable[k] = this[k]
       }
     }

diff --git a/lib/errors.js b/lib/errors.js
@@ -98,6 +98,10 @@ function attributesFromErr (err) {
 
 // ---- exports
 
+function generateErrorId () {
+  return crypto.randomBytes(16).toString('hex')
+}
+
 // Create an "error" APM event object to be sent to APM server.
 //
 // Required args:
@@ -130,7 +134,7 @@ function createAPMError (args, cb) {
   let numAsyncStepsRemaining = 0 // finish() will call cb() only when this is 0.
 
   const error = {
-    id: crypto.randomBytes(16).toString('hex'),
+    id: generateErrorId(),
     timestamp: args.timestampUs
   }
   if (args.traceContext) {
@@ -260,6 +264,7 @@ function createAPMError (args, cb) {
 }
 
 module.exports = {
+  generateErrorId,
   createAPMError,
 
   // Exported for testing.

diff --git a/lib/instrumentation/index.js b/lib/instrumentation/index.js
@@ -210,7 +210,12 @@ Instrumentation.prototype._patchModule = function (exports, name, version, enabl
 Instrumentation.prototype.addEndedTransaction = function (transaction) {
   var agent = this._agent
 
-  if (this._started) {
+  if (agent._conf.disableSend) {
+    // Save effort if disableSend=true. This one log.trace related to
+    // disableSend is included as a possible log hint to future debugging for
+    // why events are not being sent to APM server.
+    agent.logger.trace('disableSend: skip sendTransaction')
+  } else if (this._started) {
     var payload = agent._transactionFilters.process(transaction._encode())
     if (!payload) return agent.logger.debug('transaction ignored by filter %o', { trans: transaction.id, trace: transaction.traceId })
     agent.logger.debug('sending transaction %o', { trans: transaction.id, trace: transaction.traceId })
@@ -223,7 +228,9 @@ Instrumentation.prototype.addEndedTransaction = function (transaction) {
 Instrumentation.prototype.addEndedSpan = function (span) {
   var agent = this._agent
 
-  if (this._started) {
+  if (agent._conf.disableSend) {
+    // Save effort and logging if disableSend=true.
+  } else if (this._started) {
     agent.logger.debug('encoding span %o', { span: span.id, parent: span.parentId, trace: span.traceId, name: span.name, type: span.type })
     span._encode(function (err, payload) {
       if (err) {

diff --git a/lib/instrumentation/transaction.js b/lib/instrumentation/transaction.js
@@ -287,11 +287,19 @@ Transaction.prototype._setOutcomeFromHttpStatusCode = function (statusCode) {
 }
 
 Transaction.prototype._captureBreakdown = function (span) {
-  if (this.ended) return
+  if (this.ended) {
+    return
+  }
+
   const agent = this._agent
   const metrics = agent._metrics
   const conf = agent._conf
 
+  // Quick out if disableSend=true, no point in the processing time.
+  if (conf.disableSend) {
+    return
+  }
+
   // Record span data
   if (this.sampled && conf.breakdownMetrics) {
     captureBreakdown(this, {

diff --git a/lib/metrics/index.js b/lib/metrics/index.js
@@ -23,10 +23,11 @@ class Metrics {
 
   start (refTimers) {
     const metricsInterval = this[agentSymbol]._conf.metricsInterval
+    const enabled = metricsInterval !== 0 && !this[agentSymbol]._conf.disableSend
     this[registrySymbol] = new MetricsRegistry(this[agentSymbol], {
       reporterOptions: {
         defaultReportingIntervalInSeconds: metricsInterval,
-        enabled: metricsInterval !== 0,
+        enabled: enabled,
         unrefTimers: !refTimers,
         logger: new NoopLogger()
       }

diff --git a/lib/noop-transport.js b/lib/noop-transport.js
@@ -0,0 +1,47 @@
+'use strict'
+
+// A no-op (does nothing) Agent transport -- i.e. the APM server client API
+// provided by elastic-apm-http-client. This is used when `disableSend=true`.
+
+class NoopTransport {
+  config (opts) {}
+
+  addMetadataFilter (fn) {}
+
+  sendSpan (span, cb) {
+    if (cb) {
+      process.nextTick(cb)
+    }
+  }
+
+  sendTransaction (transaction, cb) {
+    if (cb) {
+      process.nextTick(cb)
+    }
+  }
+
+  sendError (_error, cb) {
+    if (cb) {
+      process.nextTick(cb)
+    }
+  }
+
+  sendMetricSet (metricset, cb) {
+    if (cb) {
+      process.nextTick(cb)
+    }
+  }
+
+  flush (cb) {
+    if (cb) {
+      process.nextTick(cb)
+    }
+  }
+
+  // Inherited from Writable, called in agent.js.
+  destroy () {}
+}
+
+module.exports = {
+  NoopTransport
+}
diff --git a/package.json b/package.json
@@ -79,7 +79,7 @@
   },
   "homepage": "https://github.com/elastic/apm-agent-nodejs",
   "dependencies": {
-    "@elastic/ecs-pino-format": "^1.1.2",
+    "@elastic/ecs-pino-format": "^1.2.0",
     "after-all-results": "^2.0.0",
     "async-cache": "^1.1.0",
     "async-value-promise": "^1.1.1",

diff --git a/test/config.js b/test/config.js
@@ -39,6 +39,7 @@ var optionFixtures = [
   ['captureSpanStackTraces', 'CAPTURE_SPAN_STACK_TRACES', true],
   ['centralConfig', 'CENTRAL_CONFIG', true],
   ['containerId', 'CONTAINER_ID'],
+  ['disableSend', 'DISABLE_SEND', false],
   ['disableInstrumentations', 'DISABLE_INSTRUMENTATIONS', []],
   ['environment', 'ENVIRONMENT', 'development'],
   ['errorMessageMaxLength', 'ERROR_MESSAGE_MAX_LENGTH', 2048],