Merge branch 'master' into move-app-redirect

docs/apm/api.asciidoc

@@ -0,0 +1,260 @@
+[role="xpack"]
+[[apm-api]]
+== API
+
+Some APM app features are provided via a REST API:
+
+* <<agent-config-api>>
+
+TIP: Kibana provides additional <<api,REST APIs>>,
+and general information on <<using-api,how to use APIs>>.
+
+////
+*******************************************************
+////
+
+[[agent-config-api]]
+=== Agent Configuration API
+
+The Agent configuration API allows you to fine-tune your APM agent configuration,
+without needing to redeploy your application.
+
+The following Agent configuration APIs are available:
+
+* <<apm-update-config>> to create or update an Agent configuration
+* <<apm-delete-config>> to delete an Agent configuration.
+* <<apm-list-config>> to list all Agent configurations.
+* <<apm-search-config>> to search for an Agent configuration.
+
+////
+*******************************************************
+////
+
+[[apm-update-config]]
+==== Create or update configuration
+
+[[apm-update-config-req]]
+===== Request
+
+`PUT /api/apm/settings/agent-configuration`
+
+[[apm-update-config-req-body]]
+===== Request body
+
+`service`::
+(required, object) Service identifying the configuration to create or update.
+
+`name` :::
+  (required, string) Name of service
+
+`environment` :::
+  (optional, string) Environment of service
+
+`settings`::
+(required) Key/value object with settings and their corresponding value.
+
+`agent_name`::
+(optional) The agent name is used by the UI to determine which settings to display.
+
+
+[[apm-update-config-example]]
+===== Example
+
+[source,console]
+--------------------------------------------------
+PUT /api/apm/settings/agent-configuration
+{
+    "service" : {
+        "name" : "frontend",
+        "environment" : "production"
+    },
+    "settings" : {
+        "transaction_sample_rate" : 0.4,
+        "capture_body" : "off",
+        "transaction_max_spans" : 500
+    },
+    "agent_name": "nodejs"
+}
+--------------------------------------------------
+
+////
+*******************************************************
+////
+
+
+[[apm-delete-config]]
+==== Delete configuration
+
+[[apm-delete-config-req]]
+===== Request
+
+`DELETE /api/apm/settings/agent-configuration`
+
+[[apm-delete-config-req-body]]
+===== Request body
+`service`::
+(required, object) Service identifying the configuration to delete
+
+`name` :::
+  (required, string) Name of service
+
+`environment` :::
+  (optional, string) Environment of service
+
+
+[[apm-delete-config-example]]
+===== Example
+
+[source,console]
+--------------------------------------------------
+DELETE /api/apm/settings/agent-configuration
+{
+    "service" : {
+        "name" : "frontend",
+        "environment": "production"
+    }
+}
+--------------------------------------------------
+
+////
+*******************************************************
+////
+
+
+[[apm-list-config]]
+==== List configuration
+
+
+[[apm-list-config-req]]
+===== Request
+
+`GET  /api/apm/settings/agent-configuration`
+
+[[apm-list-config-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+[
+  {
+    "agent_name": "go",
+    "service": {
+      "name": "opbeans-go",
+      "environment": "production"
+    },
+    "settings": {
+      "transaction_sample_rate": 1,
+      "capture_body": "off",
+      "transaction_max_spans": 200
+    },
+    "@timestamp": 1581934104843,
+    "applied_by_agent": false,
+    "etag": "1e58c178efeebae15c25c539da740d21dee422fc"
+  },
+  {
+    "agent_name": "go",
+    "service": {
+      "name": "opbeans-go"
+    },
+    "settings": {
+      "transaction_sample_rate": 1,
+      "capture_body": "off",
+      "transaction_max_spans": 300
+    },
+    "@timestamp": 1581934111727,
+    "applied_by_agent": false,
+    "etag": "3eed916d3db434d9fb7f039daa681c7a04539a64"
+  },
+  {
+    "agent_name": "nodejs",
+    "service": {
+      "name": "frontend"
+    },
+    "settings": {
+      "transaction_sample_rate": 1,
+    },
+    "@timestamp": 1582031336265,
+    "applied_by_agent": false,
+    "etag": "5080ed25785b7b19f32713681e79f46996801a5b"
+  }
+]
+--------------------------------------------------
+
+[[apm-list-config-example]]
+===== Example
+
+[source,console]
+--------------------------------------------------
+GET  /api/apm/settings/agent-configuration
+--------------------------------------------------
+
+////
+*******************************************************
+////
+
+
+[[apm-search-config]]
+==== Search configuration
+
+[[apm-search-config-req]]
+===== Request
+
+`POST /api/apm/settings/agent-configuration/search`
+
+[[apm-search-config-req-body]]
+===== Request body
+
+`service`::
+(required, object) Service identifying the configuration.
+
+`name` :::
+  (required, string) Name of service
+
+`environment` :::
+  (optional, string) Environment of service
+
+`etag`::
+(required) etag is sent by the agent to indicate the etag of the last successfully applied configuration. If the etag matches an existing configuration its `applied_by_agent` property will be set to `true`. Every time a configuration is edited `applied_by_agent` is reset to `false`.
+
+[[apm-search-config-body]]
+===== Response body
+
+[source,js]
+--------------------------------------------------
+{
+  "_index": ".apm-agent-configuration",
+  "_id": "CIaqXXABmQCdPphWj8EJ",
+  "_score": 2,
+  "_source": {
+    "agent_name": "nodejs",
+    "service": {
+      "name": "frontend"
+    },
+    "settings": {
+      "transaction_sample_rate": 1,
+    },
+    "@timestamp": 1582031336265,
+    "applied_by_agent": false,
+    "etag": "5080ed25785b7b19f32713681e79f46996801a5b"
+  }
+}
+--------------------------------------------------
+
+[[apm-search-config-example]]
+===== Example
+
+[source,console]
+--------------------------------------------------
+POST /api/apm/settings/agent-configuration/search
+{
+    "etag" : "1e58c178efeebae15c25c539da740d21dee422fc",
+    "service" : {
+        "name" : "frontend",
+        "environment": "production"
+    }
+}
+--------------------------------------------------
+
+////
+*******************************************************
+////

docs/apm/index.asciidoc

@@ -24,3 +24,5 @@ include::getting-started.asciidoc[]
 include::bottlenecks.asciidoc[]
 
 include::using-the-apm-ui.asciidoc[]
+
+include::api.asciidoc[]

docs/management/advanced-options.asciidoc

@@ -62,8 +62,6 @@ mentioned use "\_default_".
 `histogram:maxBars`:: Date histograms are not generated with more bars than the value of this property, scaling values
 when necessary.
 `history:limit`:: In fields that have history, such as query inputs, show this many recent values.
-`indexPattern:fieldMapping:lookBack`:: For index patterns containing timestamps in their names,
-look for this many recent matching patterns from which to query the field mapping.
 `indexPattern:placeholder`:: The default placeholder value to use in Management > Index Patterns > Create Index Pattern.
 `metaFields`:: Fields that exist outside of `_source`. Kibana merges these fields
 into the document when displaying it.

packages/kbn-config-schema/src/types/array_type.test.ts

@@ -85,14 +85,29 @@ test('fails if mixed types of content in array', () => {
   );
 });
 
-test('returns empty array if input is empty but type has default value', () => {
-  const type = schema.arrayOf(schema.string({ defaultValue: 'test' }));
+test('fails if sparse content in array', () => {
+  const type = schema.arrayOf(schema.string());
   expect(type.validate([])).toEqual([]);
+  expect(() => type.validate([undefined])).toThrowErrorMatchingInlineSnapshot(
+    `"[0]: sparse array are not allowed"`
+  );
 });
 
-test('returns empty array if input is empty even if type is required', () => {
-  const type = schema.arrayOf(schema.string());
+test('fails if sparse content in array if optional', () => {
+  const type = schema.arrayOf(schema.maybe(schema.string()));
+  expect(type.validate([])).toEqual([]);
+  expect(() => type.validate([undefined])).toThrowErrorMatchingInlineSnapshot(
+    `"[0]: sparse array are not allowed"`
+  );
+});
+
+test('fails if sparse content in array if nullable', () => {
+  const type = schema.arrayOf(schema.nullable(schema.string()));
   expect(type.validate([])).toEqual([]);
+  expect(type.validate([null])).toEqual([null]);
+  expect(() => type.validate([undefined])).toThrowErrorMatchingInlineSnapshot(
+    `"[0]: sparse array are not allowed"`
+  );
 });
 
 test('fails for null values if optional', () => {
@@ -102,9 +117,19 @@ test('fails for null values if optional', () => {
   );
 });
 
+test('returns empty array if input is empty but type has default value', () => {
+  const type = schema.arrayOf(schema.string({ defaultValue: 'test' }));
+  expect(type.validate([])).toEqual([]);
+});
+
+test('returns empty array if input is empty even if type is required', () => {
+  const type = schema.arrayOf(schema.string());
+  expect(type.validate([])).toEqual([]);
+});
+
 test('handles default values for undefined values', () => {
-  const type = schema.arrayOf(schema.string({ defaultValue: 'foo' }));
-  expect(type.validate([undefined])).toEqual(['foo']);
+  const type = schema.arrayOf(schema.string(), { defaultValue: ['foo'] });
+  expect(type.validate(undefined)).toEqual(['foo']);
 });
 
 test('array within array', () => {

packages/kbn-config-schema/src/types/array_type.ts

@@ -31,7 +31,7 @@ export class ArrayType<T> extends Type<T[]> {
     let schema = internals
       .array()
       .items(type.getSchema().optional())
-      .sparse();
+      .sparse(false);
 
     if (options.minSize !== undefined) {
       schema = schema.min(options.minSize);
@@ -49,6 +49,8 @@ export class ArrayType<T> extends Type<T[]> {
       case 'any.required':
       case 'array.base':
         return `expected value of type [array] but got [${typeDetect(value)}]`;
+      case 'array.sparse':
+        return `sparse array are not allowed`;
       case 'array.parse':
         return `could not parse array value from [${value}]`;
       case 'array.min':

packages/kbn-optimizer/src/optimizer/optimizer_config.ts

@@ -25,6 +25,15 @@ import { Bundle, WorkerConfig } from '../common';
 import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins';
 import { getBundles } from './get_bundles';
 
+function pickMaxWorkerCount(dist: boolean) {
+  // don't break if cpus() returns nothing, or an empty array
+  const cpuCount = Math.max(Os.cpus()?.length, 1);
+  // if we're buiding the dist then we can use more of the system's resources to get things done a little quicker
+  const maxWorkers = dist ? cpuCount - 1 : Math.ceil(cpuCount / 3);
+  // ensure we always have at least two workers
+  return Math.max(maxWorkers, 2);
+}
+
 interface Options {
   /** absolute path to root of the repo/build */
   repoRoot: string;
@@ -110,7 +119,7 @@ export class OptimizerConfig {
 
     const maxWorkerCount = process.env.KBN_OPTIMIZER_MAX_WORKERS
       ? parseInt(process.env.KBN_OPTIMIZER_MAX_WORKERS, 10)
-      : options.maxWorkerCount ?? Math.max(Math.ceil(Math.max(Os.cpus()?.length, 1) / 3), 2);
+      : options.maxWorkerCount ?? pickMaxWorkerCount(dist);
     if (typeof maxWorkerCount !== 'number' || !Number.isFinite(maxWorkerCount)) {
       throw new TypeError('worker count must be a number');
     }

packages/kbn-optimizer/src/worker/webpack.config.ts

@@ -252,6 +252,7 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) {
           cache: false,
           sourceMap: false,
           extractComments: false,
+          parallel: false,
           terserOptions: {
             compress: false,
             mangle: false,

src/dev/build/tasks/os_packages/docker_generator/resources/bin/kibana-docker

@@ -142,6 +142,7 @@ kibana_vars=(
     xpack.code.security.enableGitCertCheck
     xpack.code.security.gitHostWhitelist
     xpack.code.security.gitProtocolWhitelist
+    xpack.encryptedSavedObjects.encryptionKey
     xpack.graph.enabled
     xpack.graph.canEditDrillDownUrls
     xpack.graph.savePolicy