diff --git a/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.test.ts b/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.test.ts new file mode 100644 index 00000000000000..1098594a68f8a2 --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.test.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { getIndexStateFromClusterState } from './get_index_state_from_cluster_state'; +import { ClusterStateAPIResponse } from './types'; + +describe('getIndexStateFromClusterState', () => { + const indexName = 'indexName'; + const clusterState: ClusterStateAPIResponse = { + metadata: { + indices: {}, + cluster_coordination: {} as any, + cluster_uuid: 'test', + templates: {} as any, + }, + cluster_name: 'test', + cluster_uuid: 'test', + }; + + afterEach(() => { + clusterState.metadata.indices = {}; + }); + + it('correctly extracts state from cluster state', () => { + clusterState.metadata.indices[indexName] = { state: 'open' } as any; + clusterState.metadata.indices.aTotallyDifferentIndex = { state: 'close' } as any; + expect(getIndexStateFromClusterState(indexName, clusterState)).toBe('open'); + }); + + it('correctly extracts state from aliased index in cluster state', () => { + clusterState.metadata.indices.aTotallyDifferentName = { + state: 'close', + aliases: [indexName, 'test'], + } as any; + clusterState.metadata.indices.aTotallyDifferentName1 = { + state: 'open', + aliases: ['another', 'test'], + } as any; + + expect(getIndexStateFromClusterState(indexName, clusterState)).toBe('close'); + }); + + it('throws if the index name cannot be found in the cluster state', () => { + expect(() => getIndexStateFromClusterState(indexName, clusterState)).toThrow('not found'); + clusterState.metadata.indices.aTotallyDifferentName1 = { + state: 'open', + aliases: ['another', 'test'], + } as any; + expect(() => getIndexStateFromClusterState(indexName, clusterState)).toThrow('not found'); + }); +}); diff --git a/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.ts b/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.ts new file mode 100644 index 00000000000000..75b71fee000d4a --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/common/get_index_state_from_cluster_state.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ClusterStateAPIResponse } from './types'; + +const checkAllAliases = ( + indexName: string, + clusterState: ClusterStateAPIResponse +): 'open' | 'close' => { + for (const index of Object.values(clusterState.metadata.indices)) { + if (index.aliases?.some(alias => alias === indexName)) { + return index.state; + } + } + + throw new Error(`${indexName} not found in cluster state!`); +}; + +export const getIndexStateFromClusterState = ( + indexName: string, + clusterState: ClusterStateAPIResponse +): 'open' | 'close' => + clusterState.metadata.indices[indexName] + ? clusterState.metadata.indices[indexName].state + : checkAllAliases(indexName, clusterState); diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts index ceb3a6dd60166f..1114e889882c26 100644 --- a/x-pack/plugins/upgrade_assistant/common/types.ts +++ b/x-pack/plugins/upgrade_assistant/common/types.ts @@ -34,6 +34,13 @@ export interface QueueSettings extends SavedObjectAttributes { } export interface ReindexOptions extends SavedObjectAttributes { + /** + * Whether to treat the index as if it were closed. This instructs the + * reindex strategy to first open the index, perform reindexing and + * then close the index again. + */ + openAndClose?: boolean; + /** * Set this key to configure a reindex operation as part of a * batch to be run in series. @@ -50,7 +57,6 @@ export interface ReindexOperation extends SavedObjectAttributes { reindexTaskId: string | null; reindexTaskPercComplete: number | null; errorMessage: string | null; - // This field is only used for the singleton IndexConsumerType documents. runningReindexCount: number | null; @@ -142,6 +148,14 @@ export interface EnrichedDeprecationInfo extends DeprecationInfo { index?: string; node?: string; reindex?: boolean; + /** + * Indicate what blockers have been detected for calling reindex + * against this index. + * + * @remark + * In future this could be an array of blockers. + */ + blockerForReindexing?: 'index-closed'; // 'index-closed' can be handled automatically, but requires more resources, user should be warned } export interface UpgradeAssistantStatus { @@ -149,3 +163,42 @@ export interface UpgradeAssistantStatus { cluster: EnrichedDeprecationInfo[]; indices: EnrichedDeprecationInfo[]; } + +export interface ClusterStateIndexAPIResponse { + state: 'open' | 'close'; + settings: { + index: { + verified_before_close: string; + search: { + throttled: string; + }; + number_of_shards: string; + provided_name: string; + frozen: string; + creation_date: string; + number_of_replicas: string; + uuid: string; + version: { + created: string; + }; + }; + }; + mappings: any; + aliases: string[]; +} + +export interface ClusterStateAPIResponse { + cluster_name: string; + cluster_uuid: string; + metadata: { + cluster_uuid: string; + cluster_coordination: { + term: number; + last_committed_config: string[]; + last_accepted_config: string[]; + voting_config_exclusions: []; + }; + templates: any; + indices: { [indexName: string]: ClusterStateIndexAPIResponse }; + }; +} diff --git a/x-pack/plugins/upgrade_assistant/public/application/app_context.tsx b/x-pack/plugins/upgrade_assistant/public/application/app_context.tsx index 1ae9dabd69481b..11c88a52ea24e0 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/app_context.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/app_context.tsx @@ -3,12 +3,13 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { HttpSetup } from 'src/core/public'; +import { DocLinksStart, HttpSetup } from 'src/core/public'; import React, { createContext, useContext } from 'react'; export interface ContextValue { http: HttpSetup; isCloudEnabled: boolean; + docLinks: DocLinksStart; } export const AppContext = createContext({} as any); diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs.tsx index 43ec5554aaaeef..77ee3448cd06de 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs.tsx @@ -14,7 +14,8 @@ import { EuiTabbedContent, EuiTabbedContentTab, } from '@elastic/eui'; -import { FormattedMessage, injectI18n } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; import { HttpSetup } from 'src/core/public'; import { UpgradeAssistantStatus } from '../../../common/types'; @@ -38,9 +39,11 @@ interface TabsState { clusterUpgradeState: ClusterUpgradeState; } -type Props = ReactIntl.InjectedIntlProps & { http: HttpSetup }; +interface Props { + http: HttpSetup; +} -export class UpgradeAssistantTabsUI extends React.Component { +export class UpgradeAssistantTabs extends React.Component { constructor(props: Props) { super(props); this.state = { @@ -172,7 +175,6 @@ export class UpgradeAssistantTabsUI extends React.Component { }; private get tabs() { - const { intl } = this.props; const { loadingError, loadingState, checkupData } = this.state; const commonProps: UpgradeAssistantTabProps = { loadingError, @@ -186,24 +188,21 @@ export class UpgradeAssistantTabsUI extends React.Component { return [ { id: 'overview', - name: intl.formatMessage({ - id: 'xpack.upgradeAssistant.overviewTab.overviewTabTitle', + name: i18n.translate('xpack.upgradeAssistant.overviewTab.overviewTabTitle', { defaultMessage: 'Overview', }), content: , }, { id: 'cluster', - name: intl.formatMessage({ - id: 'xpack.upgradeAssistant.checkupTab.clusterTabLabel', + name: i18n.translate('xpack.upgradeAssistant.checkupTab.clusterTabLabel', { defaultMessage: 'Cluster', }), content: ( { }, { id: 'indices', - name: intl.formatMessage({ - id: 'xpack.upgradeAssistant.checkupTab.indicesTabLabel', + name: i18n.translate('xpack.upgradeAssistant.checkupTab.indicesTabLabel', { defaultMessage: 'Indices', }), content: ( { this.setState({ telemetryState: TelemetryState.Complete }); } } - -export const UpgradeAssistantTabs = injectI18n(UpgradeAssistantTabsUI); diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/cell.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/cell.tsx index 879bb695ca60af..6eaa0de530673f 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/cell.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/cell.tsx @@ -18,6 +18,7 @@ import { import { FormattedMessage } from '@kbn/i18n/react'; import { ReindexButton } from './reindex'; import { AppContext } from '../../../../app_context'; +import { EnrichedDeprecationInfo } from '../../../../../../common/types'; interface DeprecationCellProps { items?: Array<{ title?: string; body: string }>; @@ -26,6 +27,7 @@ interface DeprecationCellProps { headline?: string; healthColor?: string; children?: ReactNode; + reindexBlocker?: EnrichedDeprecationInfo['blockerForReindexing']; } /** @@ -38,6 +40,7 @@ export const DeprecationCell: FunctionComponent = ({ docUrl, items = [], children, + reindexBlocker, }) => (
@@ -79,7 +82,14 @@ export const DeprecationCell: FunctionComponent = ({ {reindexIndexName && ( - {({ http }) => } + {({ http, docLinks }) => ( + + )} )} diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx index 5506528a3ded09..19767c34a1b064 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/index_table.tsx @@ -11,12 +11,14 @@ import { EuiBasicTable } from '@elastic/eui'; import { injectI18n } from '@kbn/i18n/react'; import { ReindexButton } from './reindex'; import { AppContext } from '../../../../app_context'; +import { EnrichedDeprecationInfo } from '../../../../../../common/types'; const PAGE_SIZES = [10, 25, 50, 100, 250, 500, 1000]; export interface IndexDeprecationDetails { index: string; reindex: boolean; + blockerForReindexing?: EnrichedDeprecationInfo['blockerForReindexing']; details?: string; } @@ -68,9 +70,10 @@ export class IndexDeprecationTableUI extends React.Component< }, ]; - if (this.actionsColumn) { - // @ts-ignore - columns.push(this.actionsColumn); + const actionsColumn = this.generateActionsColumn(); + + if (actionsColumn) { + columns.push(actionsColumn as any); } const sorting = { @@ -134,7 +137,7 @@ export class IndexDeprecationTableUI extends React.Component< return { totalItemCount, pageSizeOptions, hidePerPageOptions: false }; } - private get actionsColumn() { + private generateActionsColumn() { // NOTE: this naive implementation assumes all indices in the table are // should show the reindex button. This should work for known usecases. const { indices } = this.props; @@ -148,7 +151,16 @@ export class IndexDeprecationTableUI extends React.Component< render(indexDep: IndexDeprecationDetails) { return ( - {({ http }) => } + {({ http, docLinks }) => { + return ( + + ); + }} ); }, diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.test.tsx index a1e173737bab06..606faf52b8e2bd 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.test.tsx @@ -23,29 +23,29 @@ describe('DeprecationList', () => { test('shows simple messages when index field is not present', () => { expect(shallow()).toMatchInlineSnapshot(` -
- - -
-`); +
+ + +
+ `); }); test('shows index deprecation when index field is present', () => { @@ -59,31 +59,33 @@ describe('DeprecationList', () => { }; const wrapper = shallow(); expect(wrapper).toMatchInlineSnapshot(` - -`); + + `); }); }); @@ -98,31 +100,31 @@ describe('DeprecationList', () => { test('shows detailed messages', () => { expect(shallow()).toMatchInlineSnapshot(` -
- - -
-`); +
+ + +
+ `); }); }); }); diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.tsx index a46bc0d12fad41..ab71972f361ea2 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/list.tsx @@ -32,6 +32,7 @@ const MessageDeprecation: FunctionComponent<{ deprecation: EnrichedDeprecationIn return ( ; + return ( + + ); }; interface IndexDeprecationProps { @@ -89,6 +96,7 @@ export const DeprecationList: FunctionComponent<{ index: dep.index!, details: dep.details, reindex: dep.reindex === true, + blockerForReindexing: dep.blockerForReindexing, })); return ; } else if (currentGroupBy === GroupByOption.index) { diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/button.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/button.tsx index 30b46e0c15213e..3738e265515a0d 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/button.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/button.tsx @@ -6,12 +6,17 @@ import { set } from 'lodash'; import React, { Fragment, ReactNode } from 'react'; +import { i18n } from '@kbn/i18n'; import { Subscription } from 'rxjs'; -import { EuiButton, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiButton, EuiLoadingSpinner, EuiText, EuiToolTip } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; -import { HttpSetup } from 'src/core/public'; -import { ReindexStatus, UIReindexOption } from '../../../../../../../common/types'; +import { DocLinksStart, HttpSetup } from 'src/core/public'; +import { + EnrichedDeprecationInfo, + ReindexStatus, + UIReindexOption, +} from '../../../../../../../common/types'; import { LoadingState } from '../../../../types'; import { ReindexFlyout } from './flyout'; import { ReindexPollingService, ReindexState } from './polling_service'; @@ -19,6 +24,8 @@ import { ReindexPollingService, ReindexState } from './polling_service'; interface ReindexButtonProps { indexName: string; http: HttpSetup; + docLinks: DocLinksStart; + reindexBlocker?: EnrichedDeprecationInfo['blockerForReindexing']; } interface ReindexButtonState { @@ -61,7 +68,7 @@ export class ReindexButton extends React.Component{buttonContent}; + return ( - {buttonContent} + {showIndexedClosedWarning ? ( + + {i18n.translate( + 'xpack.upgradeAssistant.checkupTab.reindexing.reindexButton.indexClosedToolTipDetails', + { + defaultMessage: + '"{indexName}" needs to be reindexed, but it is currently closed. The Upgrade Assistant will open, reindex and then close the index. Reindexing may take longer than usual.', + values: { indexName }, + } + )} + + } + > + {button} + + ) : ( + button + )} {flyoutVisible && ( { onConfirmInputChange: jest.fn(), startReindex: jest.fn(), cancelReindex: jest.fn(), + renderGlobalCallouts: jest.fn(), reindexState: { loadingState: LoadingState.Success, lastCompletedStep: undefined, diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.tsx index e1b8f297570781..31ddaba99a896c 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/checklist_step.tsx @@ -68,17 +68,19 @@ const buttonLabel = (status?: ReindexStatus) => { * Displays a flyout that shows the current reindexing status for a given index. */ export const ChecklistFlyoutStep: React.FunctionComponent<{ + renderGlobalCallouts: () => React.ReactNode; closeFlyout: () => void; reindexState: ReindexState; startReindex: () => void; cancelReindex: () => void; -}> = ({ closeFlyout, reindexState, startReindex, cancelReindex }) => { +}> = ({ closeFlyout, reindexState, startReindex, cancelReindex, renderGlobalCallouts }) => { const { loadingState, status, hasRequiredPrivileges } = reindexState; const loading = loadingState === LoadingState.Loading || status === ReindexStatus.inProgress; return ( + {renderGlobalCallouts()} void; cancelReindex: () => void; + docLinks: DocLinksStart; + reindexBlocker?: EnrichedDeprecationInfo['blockerForReindexing']; } interface ReindexFlyoutState { currentFlyoutStep: ReindexFlyoutStep; } +const getOpenAndCloseIndexDocLink = ({ ELASTIC_WEBSITE_URL, DOC_LINK_VERSION }: DocLinksStart) => ( + + {i18n.translate( + 'xpack.upgradeAssistant.checkupTab.reindexing.flyout.openAndCloseDocumentation', + { defaultMessage: 'documentation' } + )} + +); + +const getIndexClosedCallout = (docLinks: DocLinksStart) => ( + <> + +

+ + {i18n.translate( + 'xpack.upgradeAssistant.checkupTab.reindexing.flyout.indexClosedCallout.calloutDetails.reindexingTakesLongerEmphasis', + { defaultMessage: 'Reindexing may take longer than usual' } + )} + + ), + }} + /> +

+
+ + +); + /** * Wrapper for the contents of the flyout that manages which step of the flyout to show. */ @@ -48,14 +105,28 @@ export class ReindexFlyout extends React.Component globalCallout} closeFlyout={closeFlyout} warnings={reindexState.reindexWarnings!} advanceNextStep={this.advanceNextStep} @@ -65,6 +136,7 @@ export class ReindexFlyout extends React.Component globalCallout} closeFlyout={closeFlyout} reindexState={reindexState} startReindex={startReindex} diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx index fddbe84e284616..318d2bc7baffef 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx @@ -16,6 +16,7 @@ describe('WarningsFlyoutStep', () => { advanceNextStep: jest.fn(), warnings: [ReindexWarning.allField, ReindexWarning.booleanFields], closeFlyout: jest.fn(), + renderGlobalCallouts: jest.fn(), }; it('renders', () => { diff --git a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx index 643dd2e9b6efc6..4e296aca3d0b70 100644 --- a/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx +++ b/x-pack/plugins/upgrade_assistant/public/application/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx @@ -62,6 +62,7 @@ const WarningCheckbox: React.FunctionComponent<{ ); interface WarningsConfirmationFlyoutProps { + renderGlobalCallouts: () => React.ReactNode; closeFlyout: () => void; warnings: ReindexWarning[]; advanceNextStep: () => void; @@ -91,7 +92,7 @@ export class WarningsFlyoutStep extends React.Component< } public render() { - const { warnings, closeFlyout, advanceNextStep } = this.props; + const { warnings, closeFlyout, advanceNextStep, renderGlobalCallouts } = this.props; const { checkedIds } = this.state; // Do not allow to proceed until all checkboxes are checked. @@ -100,6 +101,7 @@ export class WarningsFlyoutStep extends React.Component< return ( + {renderGlobalCallouts()} ( `/api/upgrade_assistant/reindex/${this.indexName}` ); diff --git a/x-pack/plugins/upgrade_assistant/public/plugin.ts b/x-pack/plugins/upgrade_assistant/public/plugin.ts index 614221272dd5c4..300da4eccae156 100644 --- a/x-pack/plugins/upgrade_assistant/public/plugin.ts +++ b/x-pack/plugins/upgrade_assistant/public/plugin.ts @@ -37,8 +37,8 @@ export class UpgradeAssistantUIPlugin implements Plugin { }), order: 1000, async mount({ element }) { - const [{ i18n: i18nDep }] = await getStartServices(); - return renderApp({ element, isCloudEnabled, http, i18n: i18nDep }); + const [{ i18n: i18nDep, docLinks }] = await getStartServices(); + return renderApp({ element, isCloudEnabled, http, i18n: i18nDep, docLinks }); }, }); } diff --git a/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap b/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap index 10f66fd1fc01ac..244fc96acd1943 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap +++ b/x-pack/plugins/upgrade_assistant/server/lib/__snapshots__/es_migration_apis.test.ts.snap @@ -30,6 +30,7 @@ Object { ], "indices": Array [ Object { + "blockerForReindexing": undefined, "details": "[[type: doc, field: spins], [type: doc, field: mlockall], [type: doc, field: node_master], [type: doc, field: primary]]", "index": ".monitoring-es-6-2018.11.07", "level": "warning", @@ -38,6 +39,7 @@ Object { "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { + "blockerForReindexing": undefined, "details": "[[type: tweet, field: liked]]", "index": "twitter", "level": "warning", @@ -46,6 +48,7 @@ Object { "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { + "blockerForReindexing": undefined, "details": "[[type: index-pattern, field: notExpandable], [type: config, field: xPackMonitoring:allowReport], [type: config, field: xPackMonitoring:showBanner], [type: dashboard, field: pause], [type: dashboard, field: timeRestore]]", "index": ".kibana", "level": "warning", @@ -54,6 +57,7 @@ Object { "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { + "blockerForReindexing": undefined, "details": "[[type: doc, field: notify], [type: doc, field: created], [type: doc, field: attach_payload], [type: doc, field: met]]", "index": ".watcher-history-6-2018.11.07", "level": "warning", @@ -62,6 +66,7 @@ Object { "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { + "blockerForReindexing": undefined, "details": "[[type: doc, field: snapshot]]", "index": ".monitoring-kibana-6-2018.11.07", "level": "warning", @@ -70,6 +75,7 @@ Object { "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields", }, Object { + "blockerForReindexing": undefined, "details": "[[type: tweet, field: liked]]", "index": "twitter2", "level": "warning", diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_indices_state_check.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_indices_state_check.ts new file mode 100644 index 00000000000000..9931abf7f416c4 --- /dev/null +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_indices_state_check.ts @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IScopedClusterClient } from 'kibana/server'; +import { getIndexStateFromClusterState } from '../../common/get_index_state_from_cluster_state'; +import { ClusterStateAPIResponse } from '../../common/types'; + +type StatusCheckResult = Record; + +export const esIndicesStateCheck = async ( + dataClient: IScopedClusterClient, + indices: string[] +): Promise => { + // According to https://www.elastic.co/guide/en/elasticsearch/reference/7.6/cluster-state.html + // The response from this call is considered internal and subject to change. We have an API + // integration test for asserting that the current ES version still returns what we expect. + // This lives in x-pack/test/upgrade_assistant_integration + const clusterState: ClusterStateAPIResponse = await dataClient.callAsCurrentUser( + 'cluster.state', + { + index: indices, + metric: 'metadata', + } + ); + + const result: StatusCheckResult = {}; + + indices.forEach(index => { + result[index] = getIndexStateFromClusterState(index, clusterState); + }); + + return result; +}; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts index 4ab4227ba3e919..89571a4a18231a 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.test.ts @@ -15,9 +15,24 @@ describe('getUpgradeAssistantStatus', () => { let deprecationsResponse: DeprecationAPIResponse; const dataClient = elasticsearchServiceMock.createScopedClusterClient(); - (dataClient.callAsCurrentUser as jest.Mock).mockImplementation(async (api, { path }) => { + (dataClient.callAsCurrentUser as jest.Mock).mockImplementation(async (api, { path, index }) => { if (path === '/_migration/deprecations') { return deprecationsResponse; + } else if (api === 'cluster.state') { + return { + metadata: { + indices: { + ...index.reduce((acc: any, i: any) => { + return { + ...acc, + [i]: { + state: 'open', + }, + }; + }, {}), + }, + }, + }; } else if (api === 'indices.getMapping') { return {}; } else { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts index 68f21c1fd93b56..3381e5506f39a6 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/es_migration_apis.ts @@ -8,6 +8,8 @@ import { IScopedClusterClient } from 'src/core/server'; import { DeprecationAPIResponse } from 'src/legacy/core_plugins/elasticsearch'; import { EnrichedDeprecationInfo, UpgradeAssistantStatus } from '../../common/types'; +import { esIndicesStateCheck } from './es_indices_state_check'; + export async function getUpgradeAssistantStatus( dataClient: IScopedClusterClient, isCloudEnabled: boolean @@ -20,6 +22,19 @@ export async function getUpgradeAssistantStatus( const cluster = getClusterDeprecations(deprecations, isCloudEnabled); const indices = getCombinedIndexInfos(deprecations); + const indexNames = indices.map(({ index }) => index!); + + // If we have found deprecation information for index/indices check whether the index is + // open or closed. + if (indexNames.length) { + const indexStates = await esIndicesStateCheck(dataClient, indexNames); + + indices.forEach(indexData => { + indexData.blockerForReindexing = + indexStates[indexData.index!] === 'close' ? 'index-closed' : undefined; + }); + } + const criticalWarnings = cluster.concat(indices).filter(d => d.level === 'critical'); return { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts index f6dc471d0945d3..5722a6c29b68ff 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts @@ -88,6 +88,7 @@ const removeUnsettableSettings = (settings: FlatSettings['settings']) => 'index.routing.allocation.initial_recovery._id', 'index.version.created', 'index.version.upgraded', + 'index.verified_before_close', ]); // Use `flow` to pipe the settings through each function. diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts index 4569fdfa33a83b..0a8887083c27e9 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.test.ts @@ -51,6 +51,7 @@ describe('ReindexActions', () => { expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, { indexName: 'myIndex', newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`, + reindexOptions: undefined, status: ReindexStatus.inProgress, lastCompletedStep: ReindexStep.created, locked: null, @@ -66,6 +67,7 @@ describe('ReindexActions', () => { expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, { indexName: '.internalIndex', newIndexName: `.reindexed-v${CURRENT_MAJOR_VERSION}-internalIndex`, + reindexOptions: undefined, status: ReindexStatus.inProgress, lastCompletedStep: ReindexStep.created, locked: null, @@ -83,6 +85,7 @@ describe('ReindexActions', () => { expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, { indexName, newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`, + reindexOptions: undefined, status: ReindexStatus.inProgress, lastCompletedStep: ReindexStep.created, locked: null, @@ -98,6 +101,7 @@ describe('ReindexActions', () => { expect(client.create).toHaveBeenCalledWith(REINDEX_OP_TYPE, { indexName: `reindexed-v${PREV_MAJOR_VERSION}-myIndex`, newIndexName: `reindexed-v${CURRENT_MAJOR_VERSION}-myIndex`, + reindexOptions: undefined, status: ReindexStatus.inProgress, lastCompletedStep: ReindexStep.created, locked: null, diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts index 422e78c2f12ad9..81c8f2563a66d9 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts @@ -35,7 +35,7 @@ export interface ReindexActions { /** * Creates a new reindexOp, does not perform any pre-flight checks. * @param indexName - * @param opts Options for the reindex operation + * @param opts Additional options when creating the reindex operation */ createReindexOp(indexName: string, opts?: ReindexOptions): Promise; diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts index 886ea6761e3b73..beb7b28e05e97b 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts @@ -841,7 +841,11 @@ describe('reindexService', () => { describe('newIndexCreated', () => { const reindexOp = { id: '1', - attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.newIndexCreated }, + attributes: { + ...defaultAttributes, + lastCompletedStep: ReindexStep.newIndexCreated, + reindexOptions: { openAndClose: false }, + }, } as ReindexSavedObject; beforeEach(() => { @@ -957,7 +961,11 @@ describe('reindexService', () => { describe('reindexCompleted', () => { const reindexOp = { id: '1', - attributes: { ...defaultAttributes, lastCompletedStep: ReindexStep.reindexCompleted }, + attributes: { + ...defaultAttributes, + lastCompletedStep: ReindexStep.reindexCompleted, + reindexOptions: { openAndClose: false }, + }, } as ReindexSavedObject; it('switches aliases, sets as complete, and updates lastCompletedStep', async () => { diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts index aa91b925b744b2..4cc465e1f10b9c 100644 --- a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts +++ b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.ts @@ -14,6 +14,7 @@ import { ReindexStep, ReindexWarning, } from '../../../common/types'; + import { generateNewIndexName, getReindexWarnings, @@ -52,7 +53,7 @@ export interface ReindexService { /** * Creates a new reindex operation for a given index. * @param indexName - * @param opts + * @param opts Additional options when creating a new reindex operation */ createReindexOperation(indexName: string, opts?: ReindexOptions): Promise; @@ -314,7 +315,11 @@ export const reindexServiceFactory = ( * @param reindexOp */ const startReindexing = async (reindexOp: ReindexSavedObject) => { - const { indexName } = reindexOp.attributes; + const { indexName, reindexOptions } = reindexOp.attributes; + + if (reindexOptions?.openAndClose === true) { + await callAsUser('indices.open', { index: indexName }); + } const startReindex = (await callAsUser('reindex', { refresh: true, @@ -394,7 +399,7 @@ export const reindexServiceFactory = ( * @param reindexOp */ const switchAlias = async (reindexOp: ReindexSavedObject) => { - const { indexName, newIndexName } = reindexOp.attributes; + const { indexName, newIndexName, reindexOptions } = reindexOp.attributes; const existingAliases = ( await callAsUser('indices.getAlias', { @@ -420,6 +425,10 @@ export const reindexServiceFactory = ( throw error.cannotCreateIndex(`Index aliases could not be created.`); } + if (reindexOptions?.openAndClose === true) { + await callAsUser('indices.close', { index: indexName }); + } + return actions.updateReindexOp(reindexOp, { lastCompletedStep: ReindexStep.aliasCreated, }); @@ -647,7 +656,7 @@ export const reindexServiceFactory = ( return actions.updateReindexOp(op, { status: ReindexStatus.inProgress, - reindexOptions: opts, + reindexOptions: opts ?? op.attributes.reindexOptions, }); }); }, diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_handler.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_handler.ts index 944b4a225d4425..b7569d86795909 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_handler.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_handler.ts @@ -23,7 +23,10 @@ interface ReindexHandlerArgs { licensing: LicensingPluginSetup; headers: Record; credentialStore: CredentialStore; - enqueue?: boolean; + reindexOptions?: { + openAndClose?: boolean; + enqueue?: boolean; + }; } export const reindexHandler = async ({ @@ -34,7 +37,7 @@ export const reindexHandler = async ({ licensing, log, savedObjects, - enqueue, + reindexOptions, }: ReindexHandlerArgs): Promise => { const callAsCurrentUser = dataClient.callAsCurrentUser.bind(dataClient); const reindexActions = reindexActionsFactory(savedObjects, callAsCurrentUser); @@ -51,8 +54,11 @@ export const reindexHandler = async ({ const existingOp = await reindexService.findReindexOperation(indexName); - const opts: ReindexOptions | undefined = enqueue - ? { queueSettings: { queuedAt: Date.now() } } + const opts: ReindexOptions | undefined = reindexOptions + ? { + openAndClose: reindexOptions.openAndClose, + queueSettings: reindexOptions.enqueue ? { queuedAt: Date.now() } : undefined, + } : undefined; // If the reindexOp already exists and it's paused, resume it. Otherwise create a new one. diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts index af4f7f436ec811..dc1516ad765609 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.test.ts @@ -20,7 +20,7 @@ const mockReindexService = { resumeReindexOperation: jest.fn(), cancelReindexing: jest.fn(), }; - +jest.mock('../../lib/es_indices_state_check', () => ({ esIndicesStateCheck: jest.fn() })); jest.mock('../../lib/es_version_precheck', () => ({ versionCheckHandlerWrapper: (a: any) => a, })); @@ -39,6 +39,7 @@ import { } from '../../../common/types'; import { credentialStoreFactory } from '../../lib/reindexing/credential_store'; import { registerReindexIndicesRoutes } from './reindex_indices'; +import { esIndicesStateCheck } from '../../lib/es_indices_state_check'; /** * Since these route callbacks are so thin, these serve simply as integration tests @@ -56,6 +57,7 @@ describe('reindex API', () => { } as any; beforeEach(() => { + (esIndicesStateCheck as jest.Mock).mockResolvedValue({}); mockRouter = createMockRouter(); routeDependencies = { credentialStore, @@ -166,7 +168,9 @@ describe('reindex API', () => { ); // It called create correctly - expect(mockReindexService.createReindexOperation).toHaveBeenCalledWith('theIndex', undefined); + expect(mockReindexService.createReindexOperation).toHaveBeenCalledWith('theIndex', { + openAndClose: false, + }); // It returned the right results expect(resp.status).toEqual(200); @@ -233,7 +237,10 @@ describe('reindex API', () => { kibanaResponseFactory ); // It called resume correctly - expect(mockReindexService.resumeReindexOperation).toHaveBeenCalledWith('theIndex', undefined); + expect(mockReindexService.resumeReindexOperation).toHaveBeenCalledWith('theIndex', { + openAndClose: false, + queueSettings: undefined, + }); expect(mockReindexService.createReindexOperation).not.toHaveBeenCalled(); // It returned the right results @@ -262,6 +269,7 @@ describe('reindex API', () => { describe('POST /api/upgrade_assistant/reindex/batch', () => { const queueSettingsArg = { + openAndClose: false, queueSettings: { queuedAt: expect.any(Number) }, }; it('creates a collection of index operations', async () => { diff --git a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts index 697b73d8e10f68..0846e6c0d31d3b 100644 --- a/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts +++ b/x-pack/plugins/upgrade_assistant/server/routes/reindex_indices/reindex_indices.ts @@ -16,6 +16,7 @@ import { LicensingPluginSetup } from '../../../../licensing/server'; import { ReindexStatus } from '../../../common/types'; import { versionCheckHandlerWrapper } from '../../lib/es_version_precheck'; +import { esIndicesStateCheck } from '../../lib/es_indices_state_check'; import { reindexServiceFactory, ReindexWorker } from '../../lib/reindexing'; import { CredentialStore } from '../../lib/reindexing/credential_store'; import { reindexActionsFactory } from '../../lib/reindexing/reindex_actions'; @@ -107,6 +108,7 @@ export function registerReindexIndicesRoutes( response ) => { const { indexName } = request.params; + const indexStates = await esIndicesStateCheck(dataClient, [indexName]); try { const result = await reindexHandler({ savedObjects: savedObjectsClient, @@ -116,6 +118,7 @@ export function registerReindexIndicesRoutes( licensing, headers: request.headers, credentialStore, + reindexOptions: { openAndClose: indexStates[indexName] === 'close' }, }); // Kick the worker on this node to immediately pickup the new reindex operation. @@ -187,6 +190,7 @@ export function registerReindexIndicesRoutes( response ) => { const { indexNames } = request.body; + const indexStates = await esIndicesStateCheck(dataClient, indexNames); const results: PostBatchResponse = { enqueued: [], errors: [], @@ -201,7 +205,10 @@ export function registerReindexIndicesRoutes( licensing, headers: request.headers, credentialStore, - enqueue: true, + reindexOptions: { + openAndClose: indexStates[indexName] === 'close', + enqueue: true, + }, }); results.enqueued.push(result); } catch (e) { diff --git a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js index 196e06a61833b9..fa8c3fd99f71d6 100644 --- a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js +++ b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/index.js @@ -9,5 +9,6 @@ export default function({ loadTestFile }) { this.tags('ciGroup7'); loadTestFile(require.resolve('./reindexing')); + loadTestFile(require.resolve('./status')); }); } diff --git a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js index a99c02ffef23e7..d2cae9830d31ad 100644 --- a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js +++ b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/reindexing.js @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import { ReindexStatus, REINDEX_OP_TYPE } from '../../../plugins/upgrade_assistant/common/types'; import { generateNewIndexName } from '../../../plugins/upgrade_assistant/server/lib/reindexing/index_settings'; +import { getIndexStateFromClusterState } from '../../../plugins/upgrade_assistant/common/get_index_state_from_cluster_state'; export default function({ getService }) { const supertest = getService('supertest'); @@ -178,6 +179,8 @@ export default function({ getService }) { await es.indices.create({ index: test2 }); await es.indices.create({ index: test3 }); + await es.indices.close({ index: test1 }); + const result = await supertest .post(`/api/upgrade_assistant/reindex/batch`) .set('kbn-xsrf', 'xxx') @@ -187,6 +190,8 @@ export default function({ getService }) { expect(result.body.enqueued.length).to.equal(3); expect(result.body.errors.length).to.equal(0); + const [{ newIndexName: newTest1Name }] = result.body.enqueued; + await assertQueueState(test1, 3); await waitForReindexToComplete(test1); @@ -197,6 +202,18 @@ export default function({ getService }) { await waitForReindexToComplete(test3); await assertQueueState(undefined, 0); + + // Check that the closed index is still closed after reindexing + const clusterStateResponse = await es.cluster.state({ + index: newTest1Name, + metric: 'metadata', + }); + + const test1ReindexedState = getIndexStateFromClusterState( + newTest1Name, + clusterStateResponse + ); + expect(test1ReindexedState).to.be('close'); } finally { await cleanupReindex(test1); await cleanupReindex(test2); diff --git a/x-pack/test/upgrade_assistant_integration/upgrade_assistant/status.ts b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/status.ts new file mode 100644 index 00000000000000..f38130aa594c19 --- /dev/null +++ b/x-pack/test/upgrade_assistant_integration/upgrade_assistant/status.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../api_integration/ftr_provider_context'; +import { getIndexStateFromClusterState } from '../../../plugins/upgrade_assistant/common/get_index_state_from_cluster_state'; + +// eslint-disable-next-line import/no-default-export +export default function({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('status and _cluster/state contract', () => { + beforeEach(async () => { + await es.indices.open({ index: '7.0-data' }); + }); + + afterEach(async () => { + await es.indices.open({ index: '7.0-data' }); + }); + + // According to https://www.elastic.co/guide/en/elasticsearch/reference/7.6/cluster-state.html + // The response from this call is considered internal and subject to change. We check that + // the contract has not changed in this integration test. + it('the _cluster/state endpoint is still what we expect', async () => { + await esArchiver.load('upgrade_assistant/reindex'); + await es.indices.close({ index: '7.0-data' }); + const result = await es.cluster.state({ + index: '7.0-data', + metric: 'metadata', + }); + + try { + if (getIndexStateFromClusterState('7.0-data', result.body) === 'close') { + return; + } + } catch (e) { + expect().fail( + `Can no longer access index open/closed state. Please update Upgrade Assistant checkup. (${e.message})` + ); + return; + } + expect().fail( + `The response contract for _cluster/state metadata has changed. Please update Upgrade Assistant checkup. Received ${JSON.stringify( + result, + null, + 2 + )}. + +Expected body.metadata.indices['7.0-data'].state to be "close".` + ); + }); + }); +}