Merge branch 'master' into drilldowns-temp

docs/setup/upgrade/upgrade-migrations.asciidoc

@@ -54,6 +54,14 @@ This section highlights common causes of {kib} upgrade failures and how to preve
 ===== Corrupt saved objects
 We highly recommend testing your {kib} upgrade in a development cluster to discover and remedy problems caused by corrupt documents, especially when there are custom integrations creating saved objects in your environment. Saved objects that were corrupted through manual editing or integrations will cause migration failures with a log message like `Failed to transform document. Transform: index-pattern:7.0.0\n Doc: {...}` or `Unable to migrate the corrupt Saved Object document ...`. Corrupt documents will have to be fixed or deleted before an upgrade migration can succeed.
 
+For example, given the following error message:
+> Unable to migrate the corrupt saved object document with _id: 'marketing_space:dashboard:e3c5fc71-ac71-4805-bcab-2bcc9cc93275'. To allow migrations to proceed, please delete this document from the [.kibana_7.12.0_001] index.
+
+The following steps must be followed to allow the upgrade migration to succeed.
+Please be aware the Dashboard having ID `e3c5fc71-ac71-4805-bcab-2bcc9cc93275` belonging to the space `marketing_space` will no more be available:
+1. Delete the corrupt document with `DELETE .kibana_7.12.0_001/_doc/marketing_space:dashboard:e3c5fc71-ac71-4805-bcab-2bcc9cc93275`
+2. Restart {kib}
+
 [float]
 ===== User defined index templates that causes new `.kibana*` indices to have incompatible settings or mappings
 Matching index templates which specify `settings.refresh_interval` or `mappings` are known to interfere with {kib} upgrades.
@@ -120,4 +128,4 @@ In order to rollback after a failed upgrade migration, the saved object indices
 [[upgrade-migrations-old-indices]]
 ==== Handling old `.kibana_N` indices
 
-After migrations have completed, there will be multiple {kib} indices in {es}: (`.kibana_1`, `.kibana_2`, `.kibana_7.12.0` etc). {kib} only uses the index that the `.kibana` and `.kibana_task_manager` alias points to. The other {kib} indices can be safely deleted, but are left around as a matter of historical record, and to facilitate rolling {kib} back to a previous version.
+After migrations have completed, there will be multiple {kib} indices in {es}: (`.kibana_1`, `.kibana_2`, `.kibana_7.12.0` etc). {kib} only uses the index that the `.kibana` and `.kibana_task_manager` alias points to. The other {kib} indices can be safely deleted, but are left around as a matter of historical record, and to facilitate rolling {kib} back to a previous version.

src/core/server/http/integration_tests/core_services.test.ts

@@ -540,5 +540,50 @@ describe('http service', () => {
 
       expect(header['www-authenticate']).toEqual('Basic realm="Authorization Required"');
     });
+
+    it('provides error reason for Elasticsearch Response Errors', async () => {
+      const { http } = await root.setup();
+      const { createRouter } = http;
+      // eslint-disable-next-line prefer-const
+      let elasticsearch: InternalElasticsearchServiceStart;
+
+      esClient.ping.mockImplementation(() =>
+        elasticsearchClientMock.createErrorTransportRequestPromise(
+          new ResponseError({
+            statusCode: 404,
+            body: {
+              error: {
+                type: 'error_type',
+                reason: 'error_reason',
+              },
+            },
+            warnings: [],
+            headers: {},
+            meta: {} as any,
+          })
+        )
+      );
+
+      const router = createRouter('/new-platform');
+      router.get({ path: '/', validate: false }, async (context, req, res) => {
+        try {
+          const result = await elasticsearch.client.asScoped(req).asInternalUser.ping();
+          return res.ok({
+            body: result,
+          });
+        } catch (e) {
+          return res.badRequest({
+            body: e,
+          });
+        }
+      });
+
+      const coreStart = await root.start();
+      elasticsearch = coreStart.elasticsearch;
+
+      const { body } = await kbnTestServer.request.get(root, '/new-platform/').expect(400);
+
+      expect(body.message).toEqual('[error_type]: error_reason');
+    });
   });
 });

src/core/server/http/router/response_adapter.ts

@@ -14,6 +14,8 @@ import typeDetect from 'type-detect';
 import Boom from '@hapi/boom';
 import * as stream from 'stream';
 
+import { isResponseError as isElasticsearchResponseError } from '../../elasticsearch/client/errors';
+
 import {
   HttpResponsePayload,
   KibanaResponse,
@@ -147,6 +149,11 @@ function getErrorMessage(payload?: ResponseError): string {
     throw new Error('expected error message to be provided');
   }
   if (typeof payload === 'string') return payload;
+  // for ES response errors include nested error reason message. it doesn't contain sensitive data.
+  if (isElasticsearchResponseError(payload)) {
+    return `[${payload.message}]: ${payload.meta.body?.error?.reason}`;
+  }
+
   return getErrorMessage(payload.message);
 }
 

src/plugins/discover/public/application/components/sidebar/discover_sidebar.tsx

@@ -7,7 +7,8 @@
  */
 
 import './discover_sidebar.scss';
-import React, { useCallback, useEffect, useState, useMemo } from 'react';
+import { throttle } from 'lodash';
+import React, { useCallback, useEffect, useState, useMemo, useRef } from 'react';
 import { i18n } from '@kbn/i18n';
 import {
   EuiAccordion,
@@ -18,7 +19,9 @@ import {
   EuiSpacer,
   EuiNotificationBadge,
   EuiPageSideBar,
+  useResizeObserver,
 } from '@elastic/eui';
+
 import { isEqual, sortBy } from 'lodash';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { DiscoverField } from './discover_field';
@@ -32,6 +35,11 @@ import { FieldFilterState, getDefaultFieldFilter, setFieldFilterProp } from './l
 import { getIndexPatternFieldList } from './lib/get_index_pattern_field_list';
 import { DiscoverSidebarResponsiveProps } from './discover_sidebar_responsive';
 
+/**
+ * Default number of available fields displayed and added on scroll
+ */
+const FIELDS_PER_PAGE = 50;
+
 export interface DiscoverSidebarProps extends DiscoverSidebarResponsiveProps {
   /**
    * Current state of the field filter, filtering fields by name, type, ...
@@ -66,26 +74,36 @@ export function DiscoverSidebar({
   unmappedFieldsConfig,
 }: DiscoverSidebarProps) {
   const [fields, setFields] = useState<IndexPatternField[] | null>(null);
+  const [scrollContainer, setScrollContainer] = useState<Element | null>(null);
+  const [fieldsToRender, setFieldsToRender] = useState(FIELDS_PER_PAGE);
+  const [fieldsPerPage, setFieldsPerPage] = useState(FIELDS_PER_PAGE);
+  const availableFieldsContainer = useRef<HTMLUListElement | null>(null);
 
   useEffect(() => {
     const newFields = getIndexPatternFieldList(selectedIndexPattern, fieldCounts);
     setFields(newFields);
   }, [selectedIndexPattern, fieldCounts, hits]);
 
+  const scrollDimensions = useResizeObserver(scrollContainer);
+
   const onChangeFieldSearch = useCallback(
     (field: string, value: string | boolean | undefined) => {
       const newState = setFieldFilterProp(fieldFilter, field, value);
       setFieldFilter(newState);
+      setFieldsToRender(fieldsPerPage);
     },
-    [fieldFilter, setFieldFilter]
+    [fieldFilter, setFieldFilter, setFieldsToRender, fieldsPerPage]
   );
 
   const getDetailsByField = useCallback(
     (ipField: IndexPatternField) => getDetails(ipField, hits, columns, selectedIndexPattern),
     [hits, columns, selectedIndexPattern]
   );
 
-  const popularLimit = services.uiSettings.get(FIELDS_LIMIT_SETTING);
+  const popularLimit = useMemo(() => services.uiSettings.get(FIELDS_LIMIT_SETTING), [
+    services.uiSettings,
+  ]);
+
   const {
     selected: selectedFields,
     popular: popularFields,
@@ -112,6 +130,50 @@ export function DiscoverSidebar({
     ]
   );
 
+  const paginate = useCallback(() => {
+    const newFieldsToRender = fieldsToRender + Math.round(fieldsPerPage * 0.5);
+    setFieldsToRender(Math.max(fieldsPerPage, Math.min(newFieldsToRender, unpopularFields.length)));
+  }, [setFieldsToRender, fieldsToRender, unpopularFields, fieldsPerPage]);
+
+  useEffect(() => {
+    if (scrollContainer && unpopularFields.length && availableFieldsContainer.current) {
+      const { clientHeight, scrollHeight } = scrollContainer;
+      const isScrollable = scrollHeight > clientHeight; // there is no scrolling currently
+      const allFieldsRendered = fieldsToRender >= unpopularFields.length;
+
+      if (!isScrollable && !allFieldsRendered) {
+        // Not all available fields were rendered with the given fieldsPerPage number
+        // and no scrolling is available due to the a high zoom out factor of the browser
+        // In this case the fieldsPerPage needs to be adapted
+        const fieldsRenderedHeight = availableFieldsContainer.current.clientHeight;
+        const avgHeightPerItem = Math.round(fieldsRenderedHeight / fieldsToRender);
+        const newFieldsPerPage = Math.round(clientHeight / avgHeightPerItem) + 10;
+        if (newFieldsPerPage >= FIELDS_PER_PAGE && newFieldsPerPage !== fieldsPerPage) {
+          setFieldsPerPage(newFieldsPerPage);
+          setFieldsToRender(newFieldsPerPage);
+        }
+      }
+    }
+  }, [
+    fieldsPerPage,
+    scrollContainer,
+    unpopularFields,
+    fieldsToRender,
+    setFieldsPerPage,
+    setFieldsToRender,
+    scrollDimensions,
+  ]);
+
+  const lazyScroll = useCallback(() => {
+    if (scrollContainer) {
+      const { scrollTop, clientHeight, scrollHeight } = scrollContainer;
+      const nearBottom = scrollTop + clientHeight > scrollHeight * 0.9;
+      if (nearBottom && unpopularFields) {
+        paginate();
+      }
+    }
+  }, [paginate, scrollContainer, unpopularFields]);
+
   const fieldTypes = useMemo(() => {
     const result = ['any'];
     if (Array.isArray(fields)) {
@@ -145,12 +207,19 @@ export function DiscoverSidebar({
     return map;
   }, [fields, useNewFieldsApi, selectedFields]);
 
+  const getPaginated = useCallback(
+    (list) => {
+      return list.slice(0, fieldsToRender);
+    },
+    [fieldsToRender]
+  );
+
+  const filterChanged = useMemo(() => isEqual(fieldFilter, getDefaultFieldFilter()), [fieldFilter]);
+
   if (!selectedIndexPattern || !fields) {
     return null;
   }
 
-  const filterChanged = isEqual(fieldFilter, getDefaultFieldFilter());
-
   if (useFlyout) {
     return (
       <section
@@ -207,9 +276,18 @@ export function DiscoverSidebar({
           </form>
         </EuiFlexItem>
         <EuiFlexItem className="eui-yScroll">
-          <div>
+          <div
+            ref={(el) => {
+              if (el && !el.dataset.dynamicScroll) {
+                el.dataset.dynamicScroll = 'true';
+                setScrollContainer(el);
+              }
+            }}
+            onScroll={throttle(lazyScroll, 100)}
+            className="eui-yScroll"
+          >
             {fields.length > 0 && (
-              <>
+              <div>
                 {selectedFields &&
                 selectedFields.length > 0 &&
                 selectedFields[0].displayName !== '_source' ? (
@@ -241,11 +319,7 @@ export function DiscoverSidebar({
                       >
                         {selectedFields.map((field: IndexPatternField) => {
                           return (
-                            <li
-                              key={`field${field.name}`}
-                              data-attr-field={field.name}
-                              className="dscSidebar__item"
-                            >
+                            <li key={`field${field.name}`} data-attr-field={field.name}>
                               <DiscoverField
                                 alwaysShowActionButton={alwaysShowActionButtons}
                                 field={field}
@@ -303,11 +377,7 @@ export function DiscoverSidebar({
                       >
                         {popularFields.map((field: IndexPatternField) => {
                           return (
-                            <li
-                              key={`field${field.name}`}
-                              data-attr-field={field.name}
-                              className="dscSidebar__item"
-                            >
+                            <li key={`field${field.name}`} data-attr-field={field.name}>
                               <DiscoverField
                                 alwaysShowActionButton={alwaysShowActionButtons}
                                 field={field}
@@ -329,14 +399,11 @@ export function DiscoverSidebar({
                     className="dscFieldList dscFieldList--unpopular"
                     aria-labelledby="available_fields"
                     data-test-subj={`fieldList-unpopular`}
+                    ref={availableFieldsContainer}
                   >
-                    {unpopularFields.map((field: IndexPatternField) => {
+                    {getPaginated(unpopularFields).map((field: IndexPatternField) => {
                       return (
-                        <li
-                          key={`field${field.name}`}
-                          data-attr-field={field.name}
-                          className="dscSidebar__item"
-                        >
+                        <li key={`field${field.name}`} data-attr-field={field.name}>
                           <DiscoverField
                             alwaysShowActionButton={alwaysShowActionButtons}
                             field={field}
@@ -353,7 +420,7 @@ export function DiscoverSidebar({
                     })}
                   </ul>
                 </EuiAccordion>
-              </>
+              </div>
             )}
           </div>
         </EuiFlexItem>

src/plugins/home/public/index.ts

@@ -8,24 +8,28 @@
 
 import { PluginInitializerContext } from 'kibana/public';
 
-export {
+export type {
   FeatureCatalogueSetup,
   EnvironmentSetup,
   TutorialSetup,
   HomePublicPluginSetup,
   HomePublicPluginStart,
 } from './plugin';
-export {
+
+export { FeatureCatalogueCategory } from './services';
+
+export type {
   FeatureCatalogueEntry,
   FeatureCatalogueSolution,
-  FeatureCatalogueCategory,
   Environment,
   TutorialVariables,
   TutorialDirectoryNoticeComponent,
   TutorialDirectoryHeaderLinkComponent,
   TutorialModuleNoticeComponent,
 } from './services';
-export * from '../common/instruction_variant';
+
+export { INSTRUCTION_VARIANT, getDisplayText } from '../common/instruction_variant';
+
 import { HomePublicPlugin } from './plugin';
 
 export const plugin = (initializerContext: PluginInitializerContext) =>

src/plugins/home/public/services/environment/index.ts

@@ -6,4 +6,5 @@
  * Side Public License, v 1.
  */
 
-export { EnvironmentService, Environment, EnvironmentServiceSetup } from './environment';
+export { EnvironmentService } from './environment';
+export type { Environment, EnvironmentServiceSetup } from './environment';

src/plugins/home/public/services/feature_catalogue/index.ts

@@ -6,10 +6,10 @@
  * Side Public License, v 1.
  */
 
-export {
-  FeatureCatalogueCategory,
+export { FeatureCatalogueCategory, FeatureCatalogueRegistry } from './feature_catalogue_registry';
+
+export type {
   FeatureCatalogueEntry,
   FeatureCatalogueSolution,
-  FeatureCatalogueRegistry,
   FeatureCatalogueRegistrySetup,
 } from './feature_catalogue_registry';

src/plugins/home/public/services/index.ts

@@ -6,6 +6,23 @@
  * Side Public License, v 1.
  */
 
-export * from './feature_catalogue';
-export * from './environment';
-export * from './tutorials';
+export { FeatureCatalogueCategory, FeatureCatalogueRegistry } from './feature_catalogue';
+
+export type {
+  FeatureCatalogueEntry,
+  FeatureCatalogueSolution,
+  FeatureCatalogueRegistrySetup,
+} from './feature_catalogue';
+
+export { EnvironmentService } from './environment';
+export type { Environment, EnvironmentServiceSetup } from './environment';
+
+export { TutorialService } from './tutorials';
+
+export type {
+  TutorialVariables,
+  TutorialServiceSetup,
+  TutorialDirectoryNoticeComponent,
+  TutorialDirectoryHeaderLinkComponent,
+  TutorialModuleNoticeComponent,
+} from './tutorials';