Merge branch 'master' of github.com:elastic/kibana into implement/bui…

…ld-plugins-from-src

docs/apm/machine-learning.asciidoc

@@ -1,36 +1,61 @@
 [role="xpack"]
 [[machine-learning-integration]]
-===  integration
+=== Machine learning integration
 
 ++++
 <titleabbrev>Integrate with machine learning</titleabbrev>
 ++++
 
-The Machine Learning integration initiates a new job predefined to calculate anomaly scores on APM transaction durations.
-Jobs can be created per transaction type, and are based on the service's average response time.
+The Machine learning integration initiates a new job predefined to calculate anomaly scores on APM transaction durations.
+With this integration, you can quickly pinpoint anomalous transactions and see the health of
+any upstream and downstream services.
 
-After a machine learning job is created, results are shown in two places:
+Machine learning jobs are created per environment, and are based on a service's average response time.
+Because jobs are created at the environment level,
+you can add new services to your existing environments without the need for additional machine learning jobs.
 
-The transaction duration graph will show the expected bounds and add an annotation when the anomaly score is 75 or above.
+After a machine learning job is created, results are shown in two places:
 
+* The transaction duration chart will show the expected bounds and add an annotation when the anomaly score is 75 or above.
++
 [role="screenshot"]
 image::apm/images/apm-ml-integration.png[Example view of anomaly scores on response times in the APM app]
 
-Service maps will display a color-coded anomaly indicator based on the detected anomaly score.
-
+* Service maps will display a color-coded anomaly indicator based on the detected anomaly score.
++
 [role="screenshot"]
 image::apm/images/apm-service-map-anomaly.png[Example view of anomaly scores on service maps in the APM app]
 
 [float]
 [[create-ml-integration]]
-=== Create a new machine learning job
+=== Enable anomaly detection
+
+To enable machine learning anomaly detection:
+
+. From the Services overview, Traces overview, or Service Map tab,
+select **Anomaly detection**.
+
+. Click **Create ML Job**.
 
-To enable machine learning anomaly detection, first choose a service to monitor.
-Then, select **Integrations** > **Enable ML anomaly detection** and click **Create job**.
+. Machine learning jobs are created at the environment level.
+Select all of the service environments that you want to enable anomaly detection in.
+Anomalies will surface for all services and transaction types within the selected environments.
+
+. Click **Create Jobs**.
 
 That's it! After a few minutes, the job will begin calculating results;
-it might take additional time for results to appear on your graph.
-Jobs can be managed in *Machine Learning jobs management*.
+it might take additional time for results to appear on your service maps.
+Existing jobs can be managed in *Machine Learning jobs management*.
 
 APM specific anomaly detection wizards are also available for certain Agents.
 See the machine learning {ml-docs}/ootb-ml-jobs-apm.html[APM anomaly detection configurations] for more information.
+
+[float]
+[[warning-ml-integration]]
+=== Anomaly detection warning
+
+To make machine learning as easy as possible to set up,
+the APM app will warn you when filtered to an environment without a machine learning job.
+
+[role="screenshot"]
+image::apm/images/apm-anomaly-alert.png[Example view of anomaly alert in the APM app]

docs/development/core/server/kibana-plugin-core-server.kibanarequestevents.completed_.md

@@ -0,0 +1,18 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-core-server](./kibana-plugin-core-server.md) &gt; [KibanaRequestEvents](./kibana-plugin-core-server.kibanarequestevents.md) &gt; [completed$](./kibana-plugin-core-server.kibanarequestevents.completed_.md)
+
+## KibanaRequestEvents.completed$ property
+
+Observable that emits once if and when the request has been completely handled.
+
+<b>Signature:</b>
+
+```typescript
+completed$: Observable<void>;
+```
+
+## Remarks
+
+The request may be considered completed if: - A response has been sent to the client; or - The request was aborted.
+

docs/development/core/server/kibana-plugin-core-server.kibanarequestevents.md

@@ -17,4 +17,5 @@ export interface KibanaRequestEvents
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [aborted$](./kibana-plugin-core-server.kibanarequestevents.aborted_.md) | <code>Observable&lt;void&gt;</code> | Observable that emits once if and when the request has been aborted. |
+|  [completed$](./kibana-plugin-core-server.kibanarequestevents.completed_.md) | <code>Observable&lt;void&gt;</code> | Observable that emits once if and when the request has been completely handled. |
 

src/core/server/http/integration_tests/request.test.ts

@@ -23,6 +23,7 @@ import { HttpService } from '../http_service';
 import { contextServiceMock } from '../../context/context_service.mock';
 import { loggingSystemMock } from '../../logging/logging_system.mock';
 import { createHttpServer } from '../test_utils';
+import { schema } from '@kbn/config-schema';
 
 let server: HttpService;
 
@@ -195,6 +196,96 @@ describe('KibanaRequest', () => {
         expect(nextSpy).toHaveBeenCalledTimes(0);
         expect(completeSpy).toHaveBeenCalledTimes(1);
       });
+
+      it('does not complete before response has been sent', async () => {
+        const { server: innerServer, createRouter, registerOnPreAuth } = await server.setup(
+          setupDeps
+        );
+        const router = createRouter('/');
+
+        const nextSpy = jest.fn();
+        const completeSpy = jest.fn();
+
+        registerOnPreAuth((req, res, toolkit) => {
+          req.events.aborted$.subscribe({
+            next: nextSpy,
+            complete: completeSpy,
+          });
+          return toolkit.next();
+        });
+
+        router.post(
+          { path: '/', validate: { body: schema.any() } },
+          async (context, request, res) => {
+            expect(completeSpy).not.toHaveBeenCalled();
+            return res.ok({ body: 'ok' });
+          }
+        );
+
+        await server.start();
+
+        await supertest(innerServer.listener).post('/').send({ data: 'test' }).expect(200);
+
+        expect(nextSpy).toHaveBeenCalledTimes(0);
+        expect(completeSpy).toHaveBeenCalledTimes(1);
+      });
+    });
+
+    describe('completed$', () => {
+      it('emits once and completes when response is sent', async () => {
+        const { server: innerServer, createRouter } = await server.setup(setupDeps);
+        const router = createRouter('/');
+
+        const nextSpy = jest.fn();
+        const completeSpy = jest.fn();
+
+        router.get({ path: '/', validate: false }, async (context, req, res) => {
+          req.events.completed$.subscribe({
+            next: nextSpy,
+            complete: completeSpy,
+          });
+
+          expect(nextSpy).not.toHaveBeenCalled();
+          expect(completeSpy).not.toHaveBeenCalled();
+          return res.ok({ body: 'ok' });
+        });
+
+        await server.start();
+
+        await supertest(innerServer.listener).get('/').expect(200);
+        expect(nextSpy).toHaveBeenCalledTimes(1);
+        expect(completeSpy).toHaveBeenCalledTimes(1);
+      });
+
+      it('emits once and completes when response is aborted', async (done) => {
+        expect.assertions(2);
+        const { server: innerServer, createRouter } = await server.setup(setupDeps);
+        const router = createRouter('/');
+
+        const nextSpy = jest.fn();
+
+        router.get({ path: '/', validate: false }, async (context, req, res) => {
+          req.events.completed$.subscribe({
+            next: nextSpy,
+            complete: () => {
+              expect(nextSpy).toHaveBeenCalledTimes(1);
+              done();
+            },
+          });
+
+          expect(nextSpy).not.toHaveBeenCalled();
+          await delay(30000);
+          return res.ok({ body: 'ok' });
+        });
+
+        await server.start();
+
+        const incomingRequest = supertest(innerServer.listener)
+          .get('/')
+          // end required to send request
+          .end();
+        setTimeout(() => incomingRequest.abort(), 50);
+      });
     });
   });
 });

src/core/server/http/router/request.ts

@@ -64,6 +64,16 @@ export interface KibanaRequestEvents {
    * Observable that emits once if and when the request has been aborted.
    */
   aborted$: Observable<void>;
+
+  /**
+   * Observable that emits once if and when the request has been completely handled.
+   *
+   * @remarks
+   * The request may be considered completed if:
+   * - A response has been sent to the client; or
+   * - The request was aborted.
+   */
+  completed$: Observable<void>;
 }
 
 /**
@@ -186,11 +196,16 @@ export class KibanaRequest<
 
   private getEvents(request: Request): KibanaRequestEvents {
     const finish$ = merge(
-      fromEvent(request.raw.req, 'end'), // all data consumed
+      fromEvent(request.raw.res, 'finish'), // Response has been sent
       fromEvent(request.raw.req, 'close') // connection was closed
     ).pipe(shareReplay(1), first());
+
+    const aborted$ = fromEvent<void>(request.raw.req, 'aborted').pipe(first(), takeUntil(finish$));
+    const completed$ = merge<void, void>(finish$, aborted$).pipe(shareReplay(1), first());
+
     return {
-      aborted$: fromEvent<void>(request.raw.req, 'aborted').pipe(first(), takeUntil(finish$)),
+      aborted$,
+      completed$,
     } as const;
   }
 

src/core/server/server.api.md

@@ -1071,6 +1071,7 @@ export class KibanaRequest<Params = unknown, Query = unknown, Body = unknown, Me
 // @public
 export interface KibanaRequestEvents {
     aborted$: Observable<void>;
+    completed$: Observable<void>;
 }
 
 // @public

src/dev/build/tasks/os_packages/service_templates/sysv/etc/init.d/kibana

@@ -22,7 +22,7 @@ pidfile="/var/run/kibana/$name.pid"
 [ -r /etc/default/$name ] && . /etc/default/$name
 [ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$name
 
-export KIBANA_PATH_CONF
+export KBN_PATH_CONF
 export NODE_OPTIONS
 
 [ -z "$nice" ] && nice=0

src/dev/license_checker/config.ts

@@ -79,7 +79,7 @@ export const DEV_ONLY_LICENSE_WHITELIST = ['MPL-2.0'];
 
 // Globally overrides a license for a given package@version
 export const LICENSE_OVERRIDES = {
-  'jsts@1.1.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
+  'jsts@1.6.2': ['Eclipse Distribution License - v 1.0'], // cf. https://github.com/bjornharrtell/jsts
   '@mapbox/jsonlint-lines-primitives@2.0.2': ['MIT'], // license in readme https://github.com/tmcw/jsonlint
 
   // TODO can be removed if the https://github.com/jindw/xmldom/issues/239 is released

src/legacy/server/status/routes/api/register_stats.js

@@ -19,6 +19,7 @@
 
 import Joi from 'joi';
 import boom from 'boom';
+import { defaultsDeep } from 'lodash';
 import { i18n } from '@kbn/i18n';
 import { wrapAuthConfig } from '../../wrap_auth_config';
 import { getKibanaInfoForStats } from '../../lib';
@@ -120,10 +121,9 @@ export function registerStatsApi(usageCollection, server, config, kbnServer) {
                     },
                   };
                 } else {
-                  accum = {
-                    ...accum,
-                    [usageKey]: usage[usageKey],
-                  };
+                  // I don't think we need to it this for the above conditions, but do it for most as it will
+                  // match the behavior done in monitoring/bulk_uploader
+                  defaultsDeep(accum, { [usageKey]: usage[usageKey] });
                 }
 
                 return accum;

src/plugins/dashboard/config.ts

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+
+export const configSchema = schema.object({
+  allowByValueEmbeddables: schema.boolean({ defaultValue: false }),
+});
+
+export type ConfigSchema = TypeOf<typeof configSchema>;

src/plugins/dashboard/public/plugin.tsx

@@ -94,6 +94,10 @@ declare module '../../share/public' {
 
 export type DashboardUrlGenerator = UrlGeneratorContract<typeof DASHBOARD_APP_URL_GENERATOR>;
 
+interface DashboardFeatureFlagConfig {
+  allowByValueEmbeddables: boolean;
+}
+
 interface SetupDependencies {
   data: DataPublicPluginSetup;
   embeddable: EmbeddableSetup;
@@ -125,6 +129,7 @@ export interface DashboardStart {
     embeddableType: string;
   }) => void | undefined;
   dashboardUrlGenerator?: DashboardUrlGenerator;
+  dashboardFeatureFlagConfig: DashboardFeatureFlagConfig;
   DashboardContainerByValueRenderer: ReturnType<typeof createDashboardContainerByValueRenderer>;
 }
 
@@ -411,6 +416,7 @@ export class DashboardPlugin
       getSavedDashboardLoader: () => savedDashboardLoader,
       addEmbeddableToDashboard: this.addEmbeddableToDashboard.bind(this, core),
       dashboardUrlGenerator: this.dashboardUrlGenerator,
+      dashboardFeatureFlagConfig: this.initializerContext.config.get<DashboardFeatureFlagConfig>(),
       DashboardContainerByValueRenderer: createDashboardContainerByValueRenderer({
         factory: dashboardContainerFactory,
       }),

src/plugins/dashboard/server/index.ts

@@ -17,8 +17,16 @@
  * under the License.
  */
 
-import { PluginInitializerContext } from '../../../core/server';
+import { PluginInitializerContext, PluginConfigDescriptor } from '../../../core/server';
 import { DashboardPlugin } from './plugin';
+import { configSchema, ConfigSchema } from '../config';
+
+export const config: PluginConfigDescriptor<ConfigSchema> = {
+  exposeToBrowser: {
+    allowByValueEmbeddables: true,
+  },
+  schema: configSchema,
+};
 
 //  This exports static code and TypeScript types,
 //  as well as, Kibana Platform `plugin()` initializer.

src/plugins/discover/public/application/angular/context/api/utils/date_conversion.ts

@@ -31,15 +31,6 @@ export function extractNanos(timeFieldValue: string = ''): string {
   return fractionSeconds.length !== 9 ? fractionSeconds.padEnd(9, '0') : fractionSeconds;
 }
 
-/**
- * extract the nanoseconds as string of a given ISO formatted timestamp
- */
-export function convertIsoToNanosAsStr(isoValue: string): string {
-  const nanos = extractNanos(isoValue);
-  const millis = convertIsoToMillis(isoValue);
-  return `${millis}${nanos.substr(3, 6)}`;
-}
-
 /**
  * convert an iso formatted string to number of milliseconds since
  * 1970-01-01T00:00:00.000Z