From 4e0a5fe197038cb4891aefedb8523f6481f839c1 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 4 Oct 2017 09:21:06 +0200 Subject: [PATCH 01/13] Add ability to split shards This change adds a new `_split` API that allows to split indices into a new index with a power of two more shards that the source index. This API works alongside the `_split` API but doesn't require any shard relocation before indices can be split. The split operation is conceptually an inverse `_shrink` operation since we initialize the index with a _syntetic_ number of routing shards that are used for the consistent hashing at index time. Compared to indices created with earlier versions this might produce slightly different shard distributions but has no impact on the per-index backwards compatibility. For now, the user is required to prepare an index to be splittable by setting the split factor at index creation time. Users can decide by what factor they want to split the index ie. if an index should be splittable by into a multiple of 2 setting `index.routing_shards_factor: 1024` allows to split an index 10 times doubling the number of shards each time. This is an intermediate step until we can make this the default. This also allows us to safely backport this change to 6.x. The `_split` operation is implemented internally as a DeleteByQuery on the lucene level that is executed while the primary shards execute their initial recovery. Subsequent merges that are triggered due to this operation will not be executed immediately. All merges will be deferred unti the shards are started and will then be throttled accordingly. This change is intended for the 6.1 feature release but will not support pre-6.1 indices to be split unless these indices have been shrunk before. In that case these indices can be split backwards into their original number of shards. --- .../elasticsearch/action/ActionModule.java | 8 +- .../CreateIndexClusterStateUpdateRequest.java | 25 +- .../admin/indices/shrink/ResizeAction.java | 43 ++ ...{ShrinkRequest.java => ResizeRequest.java} | 70 ++- ...Builder.java => ResizeRequestBuilder.java} | 30 +- ...hrinkResponse.java => ResizeResponse.java} | 6 +- .../admin/indices/shrink/ResizeType.java | 27 ++ .../admin/indices/shrink/ShrinkAction.java | 10 +- .../indices/shrink/TransportResizeAction.java | 180 +++++++ .../indices/shrink/TransportShrinkAction.java | 127 +---- .../client/IndicesAdminClient.java | 19 +- .../client/support/AbstractClient.java | 20 +- .../transport/TransportProxyClient.java | 1 + .../elasticsearch/cluster/ClusterModule.java | 2 + .../cluster/metadata/IndexMetaData.java | 81 +++- .../metadata/MetaDataCreateIndexService.java | 137 ++++-- .../cluster/routing/IndexRoutingTable.java | 2 +- .../cluster/routing/OperationRouting.java | 4 +- .../decider/DiskThresholdDecider.java | 6 +- .../decider/ResizeAllocationDecider.java | 100 ++++ .../common/settings/IndexScopedSettings.java | 3 + .../common/settings/Setting.java | 5 + .../org/elasticsearch/index/mapper/Uid.java | 52 +- .../elasticsearch/index/shard/IndexShard.java | 12 +- .../index/shard/ShardSplittingQuery.java | 249 ++++++++++ .../index/shard/StoreRecovery.java | 24 +- .../admin/indices/RestShrinkIndexAction.java | 14 +- .../admin/indices/RestSplitIndexAction.java | 69 +++ .../admin/indices/create/ShrinkIndexIT.java | 16 +- .../admin/indices/create/SplitIndexIT.java | 452 ++++++++++++++++++ ...s.java => TransportResizeActionTests.java} | 20 +- .../cluster/ClusterModuleTests.java | 2 + .../metadata/IndexCreationTaskTests.java | 5 +- .../cluster/metadata/IndexMetaDataTests.java | 72 ++- .../MetaDataCreateIndexServiceTests.java | 86 +++- .../routing/OperationRoutingTests.java | 40 +- .../ResizeAllocationDeciderTests.java | 239 +++++++++ .../index/shard/ShardSplittingQueryTests.java | 193 ++++++++ .../index/shard/StoreRecoveryTests.java | 108 ++++- .../routing/PartitionedRoutingIT.java | 2 +- .../DedicatedClusterSnapshotRestoreIT.java | 2 +- .../SharedSignificantTermsTestMethods.java | 2 +- docs/reference/indices.asciidoc | 1 + docs/reference/indices/split-index.asciidoc | 158 ++++++ .../rest-api-spec/api/indices.split.json | 39 ++ .../test/indices.split/10_basic.yml | 101 ++++ .../test/indices.split/20_source_mapping.yml | 72 +++ .../cluster/routing/TestShardRouting.java | 4 + 48 files changed, 2589 insertions(+), 351 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java rename core/src/main/java/org/elasticsearch/action/admin/indices/shrink/{ShrinkRequest.java => ResizeRequest.java} (69%) rename core/src/main/java/org/elasticsearch/action/admin/indices/shrink/{ShrinkRequestBuilder.java => ResizeRequestBuilder.java} (73%) rename core/src/main/java/org/elasticsearch/action/admin/indices/shrink/{ShrinkResponse.java => ResizeResponse.java} (86%) create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java create mode 100644 core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java create mode 100644 core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java create mode 100644 core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java create mode 100644 core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java create mode 100644 core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java rename core/src/test/java/org/elasticsearch/action/admin/indices/shrink/{TransportShrinkActionTests.java => TransportResizeActionTests.java} (92%) create mode 100644 core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java create mode 100644 core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java create mode 100644 docs/reference/indices/split-index.asciidoc create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java index 86582e9b8d046..28fd3458b902a 100644 --- a/core/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/core/src/main/java/org/elasticsearch/action/ActionModule.java @@ -128,7 +128,9 @@ import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsAction; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.TransportIndicesShardStoresAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; +import org.elasticsearch.action.admin.indices.shrink.TransportResizeAction; import org.elasticsearch.action.admin.indices.shrink.TransportShrinkAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.TransportIndicesStatsAction; @@ -181,7 +183,6 @@ import org.elasticsearch.action.search.TransportMultiSearchAction; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.action.search.TransportSearchScrollAction; -import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; @@ -199,7 +200,6 @@ import org.elasticsearch.common.NamedRegistry; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.inject.multibindings.MapBinder; -import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.IndexScopedSettings; @@ -271,6 +271,7 @@ import org.elasticsearch.rest.action.admin.indices.RestRefreshAction; import org.elasticsearch.rest.action.admin.indices.RestRolloverIndexAction; import org.elasticsearch.rest.action.admin.indices.RestShrinkIndexAction; +import org.elasticsearch.rest.action.admin.indices.RestSplitIndexAction; import org.elasticsearch.rest.action.admin.indices.RestSyncedFlushAction; import org.elasticsearch.rest.action.admin.indices.RestUpdateSettingsAction; import org.elasticsearch.rest.action.admin.indices.RestUpgradeAction; @@ -324,7 +325,6 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; -import static java.util.Collections.unmodifiableList; import static java.util.Collections.unmodifiableMap; /** @@ -438,6 +438,7 @@ public void reg actions.register(IndicesShardStoresAction.INSTANCE, TransportIndicesShardStoresAction.class); actions.register(CreateIndexAction.INSTANCE, TransportCreateIndexAction.class); actions.register(ShrinkAction.INSTANCE, TransportShrinkAction.class); + actions.register(ResizeAction.INSTANCE, TransportResizeAction.class); actions.register(RolloverAction.INSTANCE, TransportRolloverAction.class); actions.register(DeleteIndexAction.INSTANCE, TransportDeleteIndexAction.class); actions.register(GetIndexAction.INSTANCE, TransportGetIndexAction.class); @@ -554,6 +555,7 @@ public void initRestHandlers(Supplier nodesInCluster) { registerHandler.accept(new RestIndicesAliasesAction(settings, restController)); registerHandler.accept(new RestCreateIndexAction(settings, restController)); registerHandler.accept(new RestShrinkIndexAction(settings, restController)); + registerHandler.accept(new RestSplitIndexAction(settings, restController)); registerHandler.accept(new RestRolloverIndexAction(settings, restController)); registerHandler.accept(new RestDeleteIndexAction(settings, restController)); registerHandler.accept(new RestCloseIndexAction(settings, restController)); diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java index a2290a5e2556e..1734c340bd4ef 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexClusterStateUpdateRequest.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.create; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ack.ClusterStateUpdateRequest; import org.elasticsearch.cluster.block.ClusterBlock; @@ -43,7 +44,8 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private final String index; private final String providedName; private final boolean updateAllTypes; - private Index shrinkFrom; + private Index recoverFrom; + private ResizeType resizeType; private IndexMetaData.State state = IndexMetaData.State.OPEN; @@ -59,7 +61,6 @@ public class CreateIndexClusterStateUpdateRequest extends ClusterStateUpdateRequ private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT; - public CreateIndexClusterStateUpdateRequest(TransportMessage originalMessage, String cause, String index, String providedName, boolean updateAllTypes) { this.originalMessage = originalMessage; @@ -99,8 +100,8 @@ public CreateIndexClusterStateUpdateRequest state(IndexMetaData.State state) { return this; } - public CreateIndexClusterStateUpdateRequest shrinkFrom(Index shrinkFrom) { - this.shrinkFrom = shrinkFrom; + public CreateIndexClusterStateUpdateRequest recoverFrom(Index recoverFrom) { + this.recoverFrom = recoverFrom; return this; } @@ -109,6 +110,11 @@ public CreateIndexClusterStateUpdateRequest waitForActiveShards(ActiveShardCount return this; } + public CreateIndexClusterStateUpdateRequest resizeType(ResizeType resizeType) { + this.resizeType = resizeType; + return this; + } + public TransportMessage originalMessage() { return originalMessage; } @@ -145,8 +151,8 @@ public Set blocks() { return blocks; } - public Index shrinkFrom() { - return shrinkFrom; + public Index recoverFrom() { + return recoverFrom; } /** True if all fields that span multiple types should be updated, false otherwise */ @@ -165,4 +171,11 @@ public String getProvidedName() { public ActiveShardCount waitForActiveShards() { return waitForActiveShards; } + + /** + * Returns the resize type or null if this is an ordinary create index request + */ + public ResizeType resizeType() { + return resizeType; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java new file mode 100644 index 0000000000000..9bd43b22eec36 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +import org.elasticsearch.action.Action; +import org.elasticsearch.client.ElasticsearchClient; + +public class ResizeAction extends Action { + + public static final ResizeAction INSTANCE = new ResizeAction(); + public static final String NAME = "indices:admin/resize"; + + private ResizeAction() { + super(NAME); + } + + @Override + public ResizeResponse newResponse() { + return new ResizeResponse(); + } + + @Override + public ResizeRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new ResizeRequestBuilder(client, this); + } +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java similarity index 69% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index 6ea58200a4500..ca4ba3716b84b 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -37,37 +38,38 @@ /** * Request class to shrink an index into a single shard */ -public class ShrinkRequest extends AcknowledgedRequest implements IndicesRequest { +public class ResizeRequest extends AcknowledgedRequest implements IndicesRequest { - public static final ObjectParser PARSER = new ObjectParser<>("shrink_request", null); + public static final ObjectParser PARSER = new ObjectParser<>("shrink_request", null); static { - PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().settings(parser.map()), + PARSER.declareField((parser, request, context) -> request.getTargetIndexRequest().settings(parser.map()), new ParseField("settings"), ObjectParser.ValueType.OBJECT); - PARSER.declareField((parser, request, context) -> request.getShrinkIndexRequest().aliases(parser.map()), + PARSER.declareField((parser, request, context) -> request.getTargetIndexRequest().aliases(parser.map()), new ParseField("aliases"), ObjectParser.ValueType.OBJECT); } - private CreateIndexRequest shrinkIndexRequest; + private CreateIndexRequest targetIndexRequest; private String sourceIndex; + private ResizeType type = ResizeType.SHRINK; - ShrinkRequest() {} + ResizeRequest() {} - public ShrinkRequest(String targetIndex, String sourceindex) { - this.shrinkIndexRequest = new CreateIndexRequest(targetIndex); - this.sourceIndex = sourceindex; + public ResizeRequest(String targetIndex, String sourceIndex) { + this.targetIndexRequest = new CreateIndexRequest(targetIndex); + this.sourceIndex = sourceIndex; } @Override public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = shrinkIndexRequest == null ? null : shrinkIndexRequest.validate(); + ActionRequestValidationException validationException = targetIndexRequest == null ? null : targetIndexRequest.validate(); if (sourceIndex == null) { validationException = addValidationError("source index is missing", validationException); } - if (shrinkIndexRequest == null) { - validationException = addValidationError("shrink index request is missing", validationException); + if (targetIndexRequest == null) { + validationException = addValidationError("target index request is missing", validationException); } - if (shrinkIndexRequest.settings().getByPrefix("index.sort.").isEmpty() == false) { - validationException = addValidationError("can't override index sort when shrinking index", validationException); + if (targetIndexRequest.settings().getByPrefix("index.sort.").isEmpty() == false) { + validationException = addValidationError("can't override index sort when resizing an index", validationException); } return validationException; } @@ -79,16 +81,24 @@ public void setSourceIndex(String index) { @Override public void readFrom(StreamInput in) throws IOException { super.readFrom(in); - shrinkIndexRequest = new CreateIndexRequest(); - shrinkIndexRequest.readFrom(in); + targetIndexRequest = new CreateIndexRequest(); + targetIndexRequest.readFrom(in); sourceIndex = in.readString(); + if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + type = in.readEnum(ResizeType.class); + } else { + type = ResizeType.SHRINK; // BWC this used to be shrink only + } } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - shrinkIndexRequest.writeTo(out); + targetIndexRequest.writeTo(out); out.writeString(sourceIndex); + if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + out.writeEnum(type); + } } @Override @@ -101,15 +111,15 @@ public IndicesOptions indicesOptions() { return IndicesOptions.lenientExpandOpen(); } - public void setShrinkIndex(CreateIndexRequest shrinkIndexRequest) { - this.shrinkIndexRequest = Objects.requireNonNull(shrinkIndexRequest, "shrink index request must not be null"); + public void setTargetIndex(CreateIndexRequest targetIndexRequest) { + this.targetIndexRequest = Objects.requireNonNull(targetIndexRequest, "target index request must not be null"); } /** * Returns the {@link CreateIndexRequest} for the shrink index */ - public CreateIndexRequest getShrinkIndexRequest() { - return shrinkIndexRequest; + public CreateIndexRequest getTargetIndexRequest() { + return targetIndexRequest; } /** @@ -128,13 +138,13 @@ public String getSourceIndex() { * non-negative integer, up to the number of copies per shard (number of replicas + 1), * to wait for the desired amount of shard copies to become active before returning. * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link ShrinkResponse#isShardsAcked()} to + * to be active before returning. Check {@link ResizeResponse#isShardsAcked()} to * determine if the requisite shard copies were all started before returning or timing out. * * @param waitForActiveShards number of active shard copies to wait on */ public void setWaitForActiveShards(ActiveShardCount waitForActiveShards) { - this.getShrinkIndexRequest().waitForActiveShards(waitForActiveShards); + this.getTargetIndexRequest().waitForActiveShards(waitForActiveShards); } /** @@ -145,4 +155,18 @@ public void setWaitForActiveShards(ActiveShardCount waitForActiveShards) { public void setWaitForActiveShards(final int waitForActiveShards) { setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } + + /** + * The type of the resize operation + */ + public void setResizeType(ResizeType type) { + this.type = Objects.requireNonNull(type); + } + + /** + * Returns the type of the resize operation + */ + public ResizeType getResizeType() { + return type; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java similarity index 73% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java index 2bd10397193d5..6d8d98c0d75f0 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkRequestBuilder.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequestBuilder.java @@ -18,31 +18,32 @@ */ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.action.Action; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.master.AcknowledgedRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; import org.elasticsearch.common.settings.Settings; -public class ShrinkRequestBuilder extends AcknowledgedRequestBuilder { - public ShrinkRequestBuilder(ElasticsearchClient client, ShrinkAction action) { - super(client, action, new ShrinkRequest()); +public class ResizeRequestBuilder extends AcknowledgedRequestBuilder { + public ResizeRequestBuilder(ElasticsearchClient client, Action action) { + super(client, action, new ResizeRequest()); } - public ShrinkRequestBuilder setTargetIndex(CreateIndexRequest request) { - this.request.setShrinkIndex(request); + public ResizeRequestBuilder setTargetIndex(CreateIndexRequest request) { + this.request.setTargetIndex(request); return this; } - public ShrinkRequestBuilder setSourceIndex(String index) { + public ResizeRequestBuilder setSourceIndex(String index) { this.request.setSourceIndex(index); return this; } - public ShrinkRequestBuilder setSettings(Settings settings) { - this.request.getShrinkIndexRequest().settings(settings); + public ResizeRequestBuilder setSettings(Settings settings) { + this.request.getTargetIndexRequest().settings(settings); return this; } @@ -55,12 +56,12 @@ public ShrinkRequestBuilder setSettings(Settings settings) { * non-negative integer, up to the number of copies per shard (number of replicas + 1), * to wait for the desired amount of shard copies to become active before returning. * Index creation will only wait up until the timeout value for the number of shard copies - * to be active before returning. Check {@link ShrinkResponse#isShardsAcked()} to + * to be active before returning. Check {@link ResizeResponse#isShardsAcked()} to * determine if the requisite shard copies were all started before returning or timing out. * * @param waitForActiveShards number of active shard copies to wait on */ - public ShrinkRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { + public ResizeRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiveShards) { this.request.setWaitForActiveShards(waitForActiveShards); return this; } @@ -70,7 +71,12 @@ public ShrinkRequestBuilder setWaitForActiveShards(ActiveShardCount waitForActiv * shard count is passed in, instead of having to first call {@link ActiveShardCount#from(int)} * to get the ActiveShardCount. */ - public ShrinkRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { + public ResizeRequestBuilder setWaitForActiveShards(final int waitForActiveShards) { return setWaitForActiveShards(ActiveShardCount.from(waitForActiveShards)); } + + public ResizeRequestBuilder setResizeType(ResizeType type) { + this.request.setResizeType(type); + return this; + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java similarity index 86% rename from core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java rename to core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java index 0c5149f6bf353..cea74ced69cfc 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkResponse.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeResponse.java @@ -21,11 +21,11 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; -public final class ShrinkResponse extends CreateIndexResponse { - ShrinkResponse() { +public final class ResizeResponse extends CreateIndexResponse { + ResizeResponse() { } - ShrinkResponse(boolean acknowledged, boolean shardsAcked, String index) { + ResizeResponse(boolean acknowledged, boolean shardsAcked, String index) { super(acknowledged, shardsAcked, index); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java new file mode 100644 index 0000000000000..bca386a9567d6 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeType.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +/** + * The type of the resize operation + */ +public enum ResizeType { + SHRINK, SPLIT; +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java index 8b5b4670e3c4d..48c23d643ba4c 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ShrinkAction.java @@ -22,7 +22,7 @@ import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; -public class ShrinkAction extends Action { +public class ShrinkAction extends Action { public static final ShrinkAction INSTANCE = new ShrinkAction(); public static final String NAME = "indices:admin/shrink"; @@ -32,12 +32,12 @@ private ShrinkAction() { } @Override - public ShrinkResponse newResponse() { - return new ShrinkResponse(); + public ResizeResponse newResponse() { + return new ResizeResponse(); } @Override - public ShrinkRequestBuilder newRequestBuilder(ElasticsearchClient client) { - return new ShrinkRequestBuilder(client, this); + public ResizeRequestBuilder newRequestBuilder(ElasticsearchClient client) { + return new ResizeRequestBuilder(client, this); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java new file mode 100644 index 0000000000000..a3b7c5b84f1bf --- /dev/null +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -0,0 +1,180 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.shrink; + +import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.stats.IndexShardStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.function.IntFunction; + +/** + * Main class to initiate resizing (shrink / split) an index into a new index + */ +public class TransportResizeAction extends TransportMasterNodeAction { + private final MetaDataCreateIndexService createIndexService; + private final Client client; + + @Inject + public TransportResizeAction(Settings settings, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetaDataCreateIndexService createIndexService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + this(settings, ResizeAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, + indexNameExpressionResolver, client); + } + + protected TransportResizeAction(Settings settings, String actionName, TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, MetaDataCreateIndexService createIndexService, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { + super(settings, actionName, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, + ResizeRequest::new); + this.createIndexService = createIndexService; + this.client = client; + } + + + @Override + protected String executor() { + // we go async right away + return ThreadPool.Names.SAME; + } + + @Override + protected ResizeResponse newResponse() { + return new ResizeResponse(); + } + + @Override + protected ClusterBlockException checkBlock(ResizeRequest request, ClusterState state) { + return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getTargetIndexRequest().index()); + } + + @Override + protected void masterOperation(final ResizeRequest resizeRequest, final ClusterState state, + final ActionListener listener) { + + // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code + final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener() { + @Override + public void onResponse(IndicesStatsResponse indicesStatsResponse) { + CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, + (i) -> { + IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i); + return shard == null ? null : shard.getPrimary().getDocs(); + }, indexNameExpressionResolver); + createIndexService.createIndex( + updateRequest, + ActionListener.wrap(response -> + listener.onResponse(new ResizeResponse(response.isAcknowledged(), response.isShardsAcked(), + updateRequest.index())), listener::onFailure + ) + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + + } + + // static for unittesting this method + static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ResizeRequest resizeRequest, final ClusterState state + , final IntFunction perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) { + final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest(); + final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index()); + final IndexMetaData metaData = state.metaData().index(sourceIndex); + final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) + .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); + int numShards = 1; + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); + } + + for (int i = 0; i < numShards; i++) { + if (resizeRequest.getResizeType() == ResizeType.SHRINK) { + Set shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); + long count = 0; + for (ShardId id : shardIds) { + DocsStats docsStats = perShardDocStats.apply(id.id()); + if (docsStats != null) { + count += docsStats.getCount(); + } + if (count > IndexWriter.MAX_DOCS) { + throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS + + "] docs - too many documents in shards " + shardIds); + } + } + } else { + Objects.requireNonNull(IndexMetaData.selectSplitShard(i, metaData, numShards)); + // we just execute this to ensure we get the right exceptions if the number of shards is wrong or less then etc. + } + } + + if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { + throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index"); + } + String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; + targetIndex.cause(cause); + Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); + settingsBuilder.put("index.number_of_shards", numShards); + targetIndex.settings(settingsBuilder); + + return new CreateIndexClusterStateUpdateRequest(targetIndex, + cause, targetIndex.index(), targetIndexName, true) + // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be + // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss + // the mappings for everything is corrupted and hard to debug + .ackTimeout(targetIndex.timeout()) + .masterNodeTimeout(targetIndex.masterNodeTimeout()) + .settings(targetIndex.settings()) + .aliases(targetIndex.aliases()) + .customs(targetIndex.customs()) + .waitForActiveShards(targetIndex.waitForActiveShards()) + .recoverFrom(metaData.getIndex()) + .resizeType(resizeRequest.getResizeType()); + } + +} diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java index 2555299709cda..9005f083d4736 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java @@ -19,143 +19,28 @@ package org.elasticsearch.action.admin.indices.shrink; -import org.apache.lucene.index.IndexWriter; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; -import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; -import org.elasticsearch.action.admin.indices.stats.IndexShardStats; -import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.shard.DocsStats; -import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Set; -import java.util.function.IntFunction; - /** - * Main class to initiate shrinking an index into a new index with a single shard + * Main class to initiate shrinking an index into a new index + * This class is only here for backwards compatibility. It will be replaced by + * TransportResizeAction in 8.0 */ -public class TransportShrinkAction extends TransportMasterNodeAction { - - private final MetaDataCreateIndexService createIndexService; - private final Client client; +public class TransportShrinkAction extends TransportResizeAction { @Inject public TransportShrinkAction(Settings settings, TransportService transportService, ClusterService clusterService, ThreadPool threadPool, MetaDataCreateIndexService createIndexService, ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, Client client) { - super(settings, ShrinkAction.NAME, transportService, clusterService, threadPool, actionFilters, indexNameExpressionResolver, - ShrinkRequest::new); - this.createIndexService = createIndexService; - this.client = client; + super(settings, ShrinkAction.NAME, transportService, clusterService, threadPool, createIndexService, actionFilters, + indexNameExpressionResolver, client); } - - @Override - protected String executor() { - // we go async right away - return ThreadPool.Names.SAME; - } - - @Override - protected ShrinkResponse newResponse() { - return new ShrinkResponse(); - } - - @Override - protected ClusterBlockException checkBlock(ShrinkRequest request, ClusterState state) { - return state.blocks().indexBlockedException(ClusterBlockLevel.METADATA_WRITE, request.getShrinkIndexRequest().index()); - } - - @Override - protected void masterOperation(final ShrinkRequest shrinkRequest, final ClusterState state, - final ActionListener listener) { - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex()); - client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener() { - @Override - public void onResponse(IndicesStatsResponse indicesStatsResponse) { - CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(shrinkRequest, state, - (i) -> { - IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i); - return shard == null ? null : shard.getPrimary().getDocs(); - }, indexNameExpressionResolver); - createIndexService.createIndex( - updateRequest, - ActionListener.wrap(response -> - listener.onResponse(new ShrinkResponse(response.isAcknowledged(), response.isShardsAcked(), updateRequest.index())), - listener::onFailure - ) - ); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - - } - - // static for unittesting this method - static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ShrinkRequest shrinkRequest, final ClusterState state - , final IntFunction perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) { - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(shrinkRequest.getSourceIndex()); - final CreateIndexRequest targetIndex = shrinkRequest.getShrinkIndexRequest(); - final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index()); - final IndexMetaData metaData = state.metaData().index(sourceIndex); - final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) - .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); - int numShards = 1; - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); - } - for (int i = 0; i < numShards; i++) { - Set shardIds = IndexMetaData.selectShrinkShards(i, metaData, numShards); - long count = 0; - for (ShardId id : shardIds) { - DocsStats docsStats = perShardDocStats.apply(id.id()); - if (docsStats != null) { - count += docsStats.getCount(); - } - if (count > IndexWriter.MAX_DOCS) { - throw new IllegalStateException("Can't merge index with more than [" + IndexWriter.MAX_DOCS - + "] docs - too many documents in shards " + shardIds); - } - } - - } - if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { - throw new IllegalArgumentException("cannot provide a routing partition size value when shrinking an index"); - } - targetIndex.cause("shrink_index"); - Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); - settingsBuilder.put("index.number_of_shards", numShards); - targetIndex.settings(settingsBuilder); - - return new CreateIndexClusterStateUpdateRequest(targetIndex, - "shrink_index", targetIndex.index(), targetIndexName, true) - // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be - // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss - // the mappings for everything is corrupted and hard to debug - .ackTimeout(targetIndex.timeout()) - .masterNodeTimeout(targetIndex.masterNodeTimeout()) - .settings(targetIndex.settings()) - .aliases(targetIndex.aliases()) - .customs(targetIndex.customs()) - .waitForActiveShards(targetIndex.waitForActiveShards()) - .shrinkFrom(metaData.getIndex()); - } - } diff --git a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java index b254039910c01..81de57f91afee 100644 --- a/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java +++ b/core/src/main/java/org/elasticsearch/client/IndicesAdminClient.java @@ -50,9 +50,6 @@ import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequestBuilder; import org.elasticsearch.action.admin.indices.exists.types.TypesExistsResponse; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequestBuilder; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.admin.indices.flush.FlushRequest; import org.elasticsearch.action.admin.indices.flush.FlushRequestBuilder; import org.elasticsearch.action.admin.indices.flush.FlushResponse; @@ -98,9 +95,9 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoreRequestBuilder; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -792,19 +789,19 @@ public interface IndicesAdminClient extends ElasticsearchClient { GetSettingsRequestBuilder prepareGetSettings(String... indices); /** - * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. + * Resize an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex); + ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex); /** - * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. + * Resize an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - ActionFuture shrinkIndex(ShrinkRequest request); + ActionFuture resizeIndex(ResizeRequest request); /** * Shrinks an index using an explicit request allowing to specify the settings, mappings and aliases of the target index of the index. */ - void shrinkIndex(ShrinkRequest request, ActionListener listener); + void resizeIndex(ResizeRequest request, ActionListener listener); /** * Swaps the index pointed to by an alias given all provided conditions are satisfied diff --git a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java index c2b813d3d659e..c0da35a307981 100644 --- a/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java +++ b/core/src/main/java/org/elasticsearch/client/support/AbstractClient.java @@ -232,10 +232,10 @@ import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresAction; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresRequest; import org.elasticsearch.action.admin.indices.shards.IndicesShardStoresResponse; -import org.elasticsearch.action.admin.indices.shrink.ShrinkAction; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequestBuilder; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequestBuilder; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsAction; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest; import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequestBuilder; @@ -1730,19 +1730,19 @@ public GetSettingsRequestBuilder prepareGetSettings(String... indices) { } @Override - public ShrinkRequestBuilder prepareShrinkIndex(String sourceIndex, String targetIndex) { - return new ShrinkRequestBuilder(this, ShrinkAction.INSTANCE).setSourceIndex(sourceIndex) + public ResizeRequestBuilder prepareResizeIndex(String sourceIndex, String targetIndex) { + return new ResizeRequestBuilder(this, ResizeAction.INSTANCE).setSourceIndex(sourceIndex) .setTargetIndex(new CreateIndexRequest(targetIndex)); } @Override - public ActionFuture shrinkIndex(ShrinkRequest request) { - return execute(ShrinkAction.INSTANCE, request); + public ActionFuture resizeIndex(ResizeRequest request) { + return execute(ResizeAction.INSTANCE, request); } @Override - public void shrinkIndex(ShrinkRequest request, ActionListener listener) { - execute(ShrinkAction.INSTANCE, request, listener); + public void resizeIndex(ResizeRequest request, ActionListener listener) { + execute(ResizeAction.INSTANCE, request, listener); } @Override diff --git a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java index 5436bef172a47..e07fab0092d0e 100644 --- a/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java +++ b/core/src/main/java/org/elasticsearch/client/transport/TransportProxyClient.java @@ -56,6 +56,7 @@ final class TransportProxyClient { ActionRequestBuilder> void execute(final Action action, final Request request, ActionListener listener) { final TransportActionNodeProxy proxy = proxies.get(action); + assert proxy != null : "no proxy found for action: " + action; nodesService.execute((n, l) -> proxy.execute(n, request, l), listener); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java index a5b7f422a9322..a4bb6a559254c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/core/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -50,6 +50,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; @@ -182,6 +183,7 @@ public static Collection createAllocationDeciders(Settings se // collect deciders by class so that we can detect duplicates Map deciders = new LinkedHashMap<>(); addAllocationDecider(deciders, new MaxRetryAllocationDecider(settings)); + addAllocationDecider(deciders, new ResizeAllocationDecider(settings)); addAllocationDecider(deciders, new ReplicaAfterPrimaryActiveAllocationDecider(settings)); addAllocationDecider(deciders, new RebalanceOnlyWhenActiveAllocationDecider(settings)); addAllocationDecider(deciders, new ClusterRebalanceAllocationDecider(settings, clusterSettings)); diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 06f203595b313..ac80415fe5862 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -195,6 +195,9 @@ static Setting buildNumberOfShardsSetting() { public static final Setting INDEX_ROUTING_PARTITION_SIZE_SETTING = Setting.intSetting(SETTING_ROUTING_PARTITION_SIZE, 1, 1, Property.IndexScope); + public static final Setting INDEX_ROUTING_SHARDS_FACTOR_SETTING = + Setting.intSetting("index.routing_shards_factor", 1, 1, Property.IndexScope); + public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; public static final Setting INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING; public static final String SETTING_READ_ONLY = "index.blocks.read_only"; @@ -455,12 +458,20 @@ public MappingMetaData mapping(String mappingType) { public static final String INDEX_SHRINK_SOURCE_UUID_KEY = "index.shrink.source.uuid"; public static final String INDEX_SHRINK_SOURCE_NAME_KEY = "index.shrink.source.name"; + public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; + public static final String INDEX_RESIZE_SOURCE_NAME_KEY = "index.resize.source.name"; public static final Setting INDEX_SHRINK_SOURCE_UUID = Setting.simpleString(INDEX_SHRINK_SOURCE_UUID_KEY); public static final Setting INDEX_SHRINK_SOURCE_NAME = Setting.simpleString(INDEX_SHRINK_SOURCE_NAME_KEY); - - - public Index getMergeSourceIndex() { - return INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index(INDEX_SHRINK_SOURCE_NAME.get(settings), INDEX_SHRINK_SOURCE_UUID.get(settings)) : null; + public static final Setting INDEX_RESIZE_SOURCE_UUID = Setting.simpleString(INDEX_RESIZE_SOURCE_UUID_KEY, + INDEX_SHRINK_SOURCE_UUID); + public static final Setting INDEX_RESIZE_SOURCE_NAME = Setting.simpleString(INDEX_RESIZE_SOURCE_NAME_KEY, + INDEX_SHRINK_SOURCE_NAME); + + public Index getResizeSourceIndex() { + return INDEX_RESIZE_SOURCE_UUID.exists(settings) || INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index + (INDEX_RESIZE_SOURCE_NAME.get + (settings), + INDEX_RESIZE_SOURCE_UUID.get(settings)) : null; } /** @@ -1006,7 +1017,6 @@ public IndexMetaData build() { throw new IllegalArgumentException("routing partition size [" + routingPartitionSize + "] should be a positive number" + " less than the number of shards [" + getRoutingNumShards() + "] for [" + index + "]"); } - // fill missing slots in inSyncAllocationIds with empty set if needed and make all entries immutable ImmutableOpenIntMap.Builder> filledInSyncAllocationIds = ImmutableOpenIntMap.builder(); for (int i = 0; i < numberOfShards; i++) { @@ -1293,12 +1303,48 @@ public int getRoutingNumShards() { /** * Returns the routing factor for this index. The default is 1. * - * @see #getRoutingFactor(IndexMetaData, int) for details + * @see #getRoutingFactor(int, int) for details */ public int getRoutingFactor() { return routingFactor; } + /** + * Returns the source shard ID to split the given target shard off + * @param shardId the id of the target shard to split into + * @param sourceIndexMetadata the source index metadata + * @param numTargetShards the total number of shards in the target index + * @return a the source shard ID to split off from + */ + public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + if (shardId >= numTargetShards) { + throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + + shardId); + } + int numSourceShards = sourceIndexMetadata.getNumberOfShards(); + if (numSourceShards > numTargetShards) { + throw new IllegalArgumentException("the number of source shards must be less that the number of target shards"); + + } + int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); + } + + /** + * Selects the source shards fro a local shard recovery. This might either be a split or a shrink operation. + * @param shardId the target shard ID to select the source shards for + * @param sourceIndexMetadata the source metadata + * @param numTargetShards the number of target shards + */ + public static Set selectRecoverFromShards(int shardId, IndexMetaData sourceIndexMetadata, int numTargetShards) { + if (sourceIndexMetadata.getNumberOfShards() > numTargetShards) { + return selectShrinkShards(shardId, sourceIndexMetadata, numTargetShards); + } else if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { + return Collections.singleton(selectSplitShard(shardId, sourceIndexMetadata, numTargetShards)); + } + throw new IllegalArgumentException("can't select recover from shards if both indices have the same number of shards"); + } + /** * Returns the source shard ids to shrink into the given shard id. * @param shardId the id of the target shard to shrink to @@ -1311,7 +1357,10 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI throw new IllegalArgumentException("the number of target shards (" + numTargetShards + ") must be greater than the shard id: " + shardId); } - int routingFactor = getRoutingFactor(sourceIndexMetadata, numTargetShards); + if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { + throw new IllegalArgumentException("the number of target shards must be less that the number of source shards"); + } + int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = new HashSet<>(routingFactor); for (int i = shardId * routingFactor; i < routingFactor*shardId + routingFactor; i++) { shards.add(new ShardId(sourceIndexMetadata.getIndex(), i)); @@ -1325,20 +1374,28 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI * {@link org.elasticsearch.cluster.routing.OperationRouting#generateShardId(IndexMetaData, String, String)} to guarantee consistent * hashing / routing of documents even if the number of shards changed (ie. a shrunk index). * - * @param sourceIndexMetadata the metadata of the source index + * @param sourceNumberOfShards the total number of shards in the source index * @param targetNumberOfShards the total number of shards in the target index * @return the routing factor for and shrunk index with the given number of target shards. * @throws IllegalArgumentException if the number of source shards is less than the number of target shards or if the source shards * are not divisible by the number of target shards. */ - public static int getRoutingFactor(IndexMetaData sourceIndexMetadata, int targetNumberOfShards) { - int sourceNumberOfShards = sourceIndexMetadata.getNumberOfShards(); + public static int getRoutingFactor(int sourceNumberOfShards, int targetNumberOfShards) { if (sourceNumberOfShards < targetNumberOfShards) { - throw new IllegalArgumentException("the number of target shards must be less that the number of source shards"); + int spare = sourceNumberOfShards; + sourceNumberOfShards = targetNumberOfShards; + targetNumberOfShards = spare; } + int factor = sourceNumberOfShards / targetNumberOfShards; if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { - throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a multiple of [" + if (sourceNumberOfShards < targetNumberOfShards) { + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + "factor of [" + + targetNumberOfShards + "]"); + } + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + "multiple of [" + targetNumberOfShards + "]"); } return factor; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 643987862ff2f..4f26fac16a10b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -31,6 +31,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; @@ -116,7 +117,6 @@ public class MetaDataCreateIndexService extends AbstractComponent { private final IndexScopedSettings indexScopedSettings; private final ActiveShardsObserver activeShardsObserver; private final NamedXContentRegistry xContentRegistry; - private final ThreadPool threadPool; @Inject public MetaDataCreateIndexService(Settings settings, ClusterService clusterService, @@ -132,7 +132,6 @@ public MetaDataCreateIndexService(Settings settings, ClusterService clusterServi this.env = env; this.indexScopedSettings = indexScopedSettings; this.activeShardsObserver = new ActiveShardsObserver(settings, clusterService, threadPool); - this.threadPool = threadPool; this.xContentRegistry = xContentRegistry; } @@ -298,9 +297,9 @@ public ClusterState execute(ClusterState currentState) throws Exception { customs.put(entry.getKey(), entry.getValue()); } - final Index shrinkFromIndex = request.shrinkFrom(); + final Index recoverFromIndex = request.recoverFrom(); - if (shrinkFromIndex == null) { + if (recoverFromIndex == null) { // apply templates, merging the mappings into the request mapping if exists for (IndexTemplateMetaData template : templates) { templateNames.add(template.getName()); @@ -351,7 +350,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { } } Settings.Builder indexSettingsBuilder = Settings.builder(); - if (shrinkFromIndex == null) { + if (recoverFromIndex == null) { // apply templates, here, in reverse order, since first ones are better matching for (int i = templates.size() - 1; i >= 0; i--) { indexSettingsBuilder.put(templates.get(i).settings()); @@ -383,28 +382,34 @@ public ClusterState execute(ClusterState currentState) throws Exception { final IndexMetaData.Builder tmpImdBuilder = IndexMetaData.builder(request.index()); final int routingNumShards; - if (shrinkFromIndex == null) { - routingNumShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(indexSettingsBuilder.build()); + if (recoverFromIndex == null) { + Settings idxSettings = indexSettingsBuilder.build(); + int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); + int routingShardsFactor = IndexMetaData.INDEX_ROUTING_SHARDS_FACTOR_SETTING.get(idxSettings); + // we multiply the routing shares in order to split the shard going forward. + // implementation wise splitting shards is really just an inverted shrink operation. + routingNumShards = numShards * routingShardsFactor; } else { - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex); + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); routingNumShards = sourceMetaData.getRoutingNumShards(); } tmpImdBuilder.setRoutingNumShards(routingNumShards); - if (shrinkFromIndex != null) { - prepareShrinkIndexSettings( - currentState, mappings.keySet(), indexSettingsBuilder, shrinkFromIndex, request.index()); + if (recoverFromIndex != null) { + assert request.resizeType() != null; + prepareResizeIndexSettings( + currentState, mappings.keySet(), indexSettingsBuilder, recoverFromIndex, request.index(), request.resizeType()); } final Settings actualIndexSettings = indexSettingsBuilder.build(); tmpImdBuilder.settings(actualIndexSettings); - if (shrinkFromIndex != null) { + if (recoverFromIndex != null) { /* * We need to arrange that the primary term on all the shards in the shrunken index is at least as large as * the maximum primary term on all the shards in the source index. This ensures that we have correct * document-level semantics regarding sequence numbers in the shrunken index. */ - final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(shrinkFromIndex); + final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); final long primaryTerm = IntStream .range(0, sourceMetaData.getNumberOfShards()) @@ -439,7 +444,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { throw e; } - if (request.shrinkFrom() == null) { + if (request.recoverFrom() == null) { // now that the mapping is merged we can validate the index sort. // we cannot validate for index shrinking since the mapping is empty // at this point. The validation will take place later in the process @@ -606,35 +611,16 @@ List getIndexSettingsValidationErrors(Settings settings) { static List validateShrinkIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { - if (state.metaData().hasIndex(targetIndexName)) { - throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex()); - } - final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex); - if (sourceMetaData == null) { - throw new IndexNotFoundException(sourceIndex); - } - // ensure index is read-only - if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { - throw new IllegalStateException("index " + sourceIndex + " must be read-only to shrink index. use \"index.blocks.write=true\""); + IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); + + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + IndexMetaData.selectShrinkShards(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } if (sourceMetaData.getNumberOfShards() == 1) { throw new IllegalArgumentException("can't shrink an index with only one shard"); } - - if ((targetIndexMappingsTypes.size() > 1 || - (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) { - throw new IllegalArgumentException("mappings are not allowed when shrinking indices" + - ", all mappings are copied from the source index"); - } - - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - // this method applies all necessary checks ie. if the target shards are less than the source shards - // of if the source shards are divisible by the number of target shards - IndexMetaData.getRoutingFactor(sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); - } - // now check that index is all on one node final IndexRoutingTable table = state.routingTable().index(sourceIndex); Map nodesToNumRouting = new HashMap<>(); @@ -657,27 +643,80 @@ static List validateShrinkIndex(ClusterState state, String sourceIndex, return nodesToAllocateOn; } - static void prepareShrinkIndexSettings(ClusterState currentState, Set mappingKeys, Settings.Builder indexSettingsBuilder, Index shrinkFromIndex, String shrinkIntoName) { - final IndexMetaData sourceMetaData = currentState.metaData().index(shrinkFromIndex.getName()); + static void validateSplitIndex(ClusterState state, String sourceIndex, + Set targetIndexMappingsTypes, String targetIndexName, + Settings targetIndexSettings) { + IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); + + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + } + if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { + // ensure we have a single type. + throw new IllegalStateException("source index created version is too old to apply a split operation"); + } + + } + + static IndexMetaData validateResize(ClusterState state, String sourceIndex, + Set targetIndexMappingsTypes, String targetIndexName, + Settings targetIndexSettings) { + if (state.metaData().hasIndex(targetIndexName)) { + throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex()); + } + final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex); + if (sourceMetaData == null) { + throw new IndexNotFoundException(sourceIndex); + } + // ensure index is read-only + if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) { + throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\""); + } + + if ((targetIndexMappingsTypes.size() > 1 || + (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) { + throw new IllegalArgumentException("mappings are not allowed when resizing indices" + + ", all mappings are copied from the source index"); + } + + if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { + // this method applies all necessary checks ie. if the target shards are less than the source shards + // of if the source shards are divisible by the number of target shards + IndexMetaData.getRoutingFactor(sourceMetaData.getNumberOfShards(), + IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); + } + return sourceMetaData; + } + + static void prepareResizeIndexSettings(ClusterState currentState, Set mappingKeys, Settings.Builder indexSettingsBuilder, + Index resizeSourceIndex, String resizeIntoName, ResizeType type) { + final IndexMetaData sourceMetaData = currentState.metaData().index(resizeSourceIndex.getName()); + if (type == ResizeType.SHRINK) { + final List nodesToAllocateOn = validateShrinkIndex(currentState, resizeSourceIndex.getName(), + mappingKeys, resizeIntoName, indexSettingsBuilder.build()); + indexSettingsBuilder + // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away + // once we are allocated. + .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", + Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) + // we only try once and then give up with a shrink index + .put("index.allocation.max_retries", 1); + } else if (type == ResizeType.SPLIT) { + validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); + } else { + throw new IllegalStateException("unknown resize type is " + type); + } - final List nodesToAllocateOn = validateShrinkIndex(currentState, shrinkFromIndex.getName(), - mappingKeys, shrinkIntoName, indexSettingsBuilder.build()); final Predicate sourceSettingsPredicate = (s) -> s.startsWith("index.similarity.") || s.startsWith("index.analysis.") || s.startsWith("index.sort."); indexSettingsBuilder - // we use "i.r.a.initial_recovery" rather than "i.r.a.require|include" since we want the replica to allocate right away - // once we are allocated. - .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", - Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) - // we only try once and then give up with a shrink index - .put("index.allocation.max_retries", 1) // now copy all similarity / analysis / sort settings - this overrides all settings from the user unless they // wanna add extra settings .put(IndexMetaData.SETTING_VERSION_CREATED, sourceMetaData.getCreationVersion()) .put(IndexMetaData.SETTING_VERSION_UPGRADED, sourceMetaData.getUpgradedVersion()) .put(sourceMetaData.getSettings().filter(sourceSettingsPredicate)) .put(IndexMetaData.SETTING_ROUTING_PARTITION_SIZE, sourceMetaData.getRoutingPartitionSize()) - .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), shrinkFromIndex.getName()) - .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), shrinkFromIndex.getUUID()); + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java index 5a0bd0d426313..5a4e0c78414dd 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/IndexRoutingTable.java @@ -411,7 +411,7 @@ private Builder initializeEmpty(IndexMetaData indexMetaData, UnassignedInfo unas if (indexMetaData.inSyncAllocationIds(shardNumber).isEmpty() == false) { // we have previous valid copies for this shard. use them for recovery primaryRecoverySource = StoreRecoverySource.EXISTING_STORE_INSTANCE; - } else if (indexMetaData.getMergeSourceIndex() != null) { + } else if (indexMetaData.getResizeSourceIndex() != null) { // this is a new index but the initial shards should merged from another index primaryRecoverySource = LocalShardsRecoverySource.INSTANCE; } else { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index 296eca476a6c5..76085523e5a0b 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -267,7 +267,7 @@ public ShardId shardId(ClusterState clusterState, String index, String id, @Null return new ShardId(indexMetaData.getIndex(), generateShardId(indexMetaData, id, routing)); } - static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) { + public static int generateShardId(IndexMetaData indexMetaData, @Nullable String id, @Nullable String routing) { final String effectiveRouting; final int partitionOffset; @@ -293,7 +293,7 @@ private static int calculateScaledShardId(IndexMetaData indexMetaData, String ef // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size // of original index to hash documents - return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor(); + return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor(); } } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java index 56663be1ef427..2a323af5f8435 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/DiskThresholdDecider.java @@ -403,14 +403,14 @@ private Decision earlyTerminate(RoutingAllocation allocation, ImmutableOpenMap shardIds = IndexMetaData.selectShrinkShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); + final Set shardIds = IndexMetaData.selectRecoverFromShards(shard.id(), sourceIndexMeta, metaData.getNumberOfShards()); for (IndexShardRoutingTable shardRoutingTable : allocation.routingTable().index(mergeSourceIndex.getName())) { if (shardIds.contains(shardRoutingTable.shardId())) { targetShardSize += info.getShardSize(shardRoutingTable.primaryShard(), 0); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java new file mode 100644 index 0000000000000..94855ac8261d2 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.cluster.routing.allocation.decider; + +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingNode; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; +import org.elasticsearch.common.settings.Setting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.index.shard.ShardId; + +import java.util.Set; + +/** + * An allocation decider that ensures we allocate the shards of a target index for resize operations next to the source primaries + * // TODO add tests!! + */ +public class ResizeAllocationDecider extends AllocationDecider { + + public static final String NAME = "resize"; + + /** + * Initializes a new {@link ResizeAllocationDecider} + * + * @param settings {@link Settings} used by this {@link AllocationDecider} + */ + public ResizeAllocationDecider(Settings settings) { + super(settings); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocation) { + return canAllocate(shardRouting, null, allocation); + } + + @Override + public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo(); + if (unassignedInfo != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { + // we only make decisions here if we have no unassigned info and we have to recover from another index ie. split / shrink + final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index()); + Index resizeSourceIndex = indexMetaData.getResizeSourceIndex(); + assert resizeSourceIndex != null; + try { + IndexMetaData sourceIndexMetaData = allocation.metaData().getIndexSafe(resizeSourceIndex); + if (indexMetaData.getNumberOfShards() < sourceIndexMetaData.getNumberOfShards()) { + // this only handles splits so far. + return Decision.ALWAYS; + } + ShardId shardId = IndexMetaData.selectSplitShard(shardRouting.id(), sourceIndexMetaData, indexMetaData.getNumberOfShards()); + ShardRouting sourceShardRouting = allocation.routingTable().shardRoutingTable(shardId).primaryShard(); + if (sourceShardRouting.active() == false) { + return allocation.decision(Decision.NO, NAME, "source primary shard [%s] is not active", sourceShardRouting.shardId()); + } + if (node != null) { // we might get called from the 2 param canAllocate method.. + if (sourceShardRouting.currentNodeId().equals(node.nodeId())) { + return allocation.decision(Decision.YES, NAME, "source primary is allocated on this node"); + } else { + return allocation.decision(Decision.NO, NAME, "source primary is allocated on another node"); + } + } else { + return allocation.decision(Decision.YES, NAME, "source primary is active"); + } + } catch (IndexNotFoundException ex) { + return allocation.decision(Decision.NO, NAME, "resize source index [%s] doesn't exists", resizeSourceIndex.toString()); + } + } + return super.canAllocate(shardRouting, node, allocation); + } + + @Override + public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { + assert shardRouting.primary() : "must not call canForceAllocatePrimary on a non-primary shard " + shardRouting; + // check if we have passed the maximum retry threshold through canAllocate, + // if so, we don't want to force the primary allocation here + return canAllocate(shardRouting, node, allocation); + } +} diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 9d4d30b066f1f..bb120ca31aa24 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -70,6 +70,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING, + IndexMetaData.INDEX_ROUTING_SHARDS_FACTOR_SETTING, IndexMetaData.INDEX_READ_ONLY_SETTING, IndexMetaData.INDEX_BLOCKS_READ_SETTING, IndexMetaData.INDEX_BLOCKS_WRITE_SETTING, @@ -199,6 +200,8 @@ protected boolean isPrivateSetting(String key) { case MergePolicyConfig.INDEX_MERGE_ENABLED: case IndexMetaData.INDEX_SHRINK_SOURCE_UUID_KEY: case IndexMetaData.INDEX_SHRINK_SOURCE_NAME_KEY: + case IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY: + case IndexMetaData.INDEX_RESIZE_SOURCE_NAME_KEY: case IndexSettings.INDEX_MAPPING_SINGLE_TYPE_SETTING_KEY: // this was settable in 5.x but not anymore in 6.x so we have to preserve the value ie. make it read-only // this can be removed in later versions diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index ee6e422e82676..c30d5c45d737c 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -51,6 +51,7 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.IntConsumer; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -915,6 +916,10 @@ public static Setting simpleString(String key, Property... properties) { return new Setting<>(key, s -> "", Function.identity(), properties); } + public static Setting simpleString(String key, Setting fallback, Property... properties) { + return new Setting<>(key, fallback, Function.identity(), properties); + } + public static Setting simpleString(String key, Validator validator, Property... properties) { return new Setting<>(new SimpleKey(key), null, s -> "", Function.identity(), validator, properties); } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java index 4b5ed5fd3cd92..dae320511e5dd 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java @@ -22,8 +22,10 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lucene.BytesRefs; +import java.io.ByteArrayInputStream; import java.util.Arrays; import java.util.Base64; import java.util.Collection; @@ -244,16 +246,16 @@ public static BytesRef encodeId(String id) { } } - private static String decodeNumericId(byte[] idBytes) { - assert Byte.toUnsignedInt(idBytes[0]) == NUMERIC; - int length = (idBytes.length - 1) * 2; + private static String decodeNumericId(byte[] idBytes, int offset, int len) { + assert Byte.toUnsignedInt(idBytes[offset]) == NUMERIC; + int length = (len - 1) * 2; char[] chars = new char[length]; - for (int i = 1; i < idBytes.length; ++i) { - final int b = Byte.toUnsignedInt(idBytes[i]); + for (int i = 1; i < len; ++i) { + final int b = Byte.toUnsignedInt(idBytes[offset + i]); final int b1 = (b >>> 4); final int b2 = b & 0x0f; chars[(i - 1) * 2] = (char) (b1 + '0'); - if (i == idBytes.length - 1 && b2 == 0x0f) { + if (i == len - 1 && b2 == 0x0f) { length--; break; } @@ -262,15 +264,17 @@ private static String decodeNumericId(byte[] idBytes) { return new String(chars, 0, length); } - private static String decodeUtf8Id(byte[] idBytes) { - assert Byte.toUnsignedInt(idBytes[0]) == UTF8; - return new BytesRef(idBytes, 1, idBytes.length - 1).utf8ToString(); + private static String decodeUtf8Id(byte[] idBytes, int offset, int length) { + assert Byte.toUnsignedInt(idBytes[offset]) == UTF8; + return new BytesRef(idBytes, offset + 1, length - 1).utf8ToString(); } - private static String decodeBase64Id(byte[] idBytes) { - assert Byte.toUnsignedInt(idBytes[0]) <= BASE64_ESCAPE; - if (Byte.toUnsignedInt(idBytes[0]) == BASE64_ESCAPE) { - idBytes = Arrays.copyOfRange(idBytes, 1, idBytes.length); + private static String decodeBase64Id(byte[] idBytes, int offset, int length) { + assert Byte.toUnsignedInt(idBytes[offset]) <= BASE64_ESCAPE; + if (Byte.toUnsignedInt(idBytes[offset]) == BASE64_ESCAPE) { + idBytes = Arrays.copyOfRange(idBytes, offset + 1, length); + } else if (idBytes.length != length || offset != 0) { + idBytes = Arrays.copyOfRange(idBytes, offset, length); } return Base64.getUrlEncoder().withoutPadding().encodeToString(idBytes); } @@ -278,17 +282,23 @@ private static String decodeBase64Id(byte[] idBytes) { /** Decode an indexed id back to its original form. * @see #encodeId */ public static String decodeId(byte[] idBytes) { - if (idBytes.length == 0) { + return decodeId(idBytes, 0, idBytes.length); + } + + /** Decode an indexed id back to its original form. + * @see #encodeId */ + public static String decodeId(byte[] idBytes, int offset, int length) { + if (length == 0) { throw new IllegalArgumentException("Ids can't be empty"); } - final int magicChar = Byte.toUnsignedInt(idBytes[0]); + final int magicChar = Byte.toUnsignedInt(idBytes[offset]); switch (magicChar) { - case NUMERIC: - return decodeNumericId(idBytes); - case UTF8: - return decodeUtf8Id(idBytes); - default: - return decodeBase64Id(idBytes); + case NUMERIC: + return decodeNumericId(idBytes, offset, length); + case UTF8: + return decodeUtf8Id(idBytes, offset, length); + default: + return decodeBase64Id(idBytes, offset, length); } } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 9f0f6b84b459f..4a5c8c4ad7661 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2026,13 +2026,15 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService break; case LOCAL_SHARDS: final IndexMetaData indexMetaData = indexSettings().getIndexMetaData(); - final Index mergeSourceIndex = indexMetaData.getMergeSourceIndex(); + final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex(); final List startedShards = new ArrayList<>(); final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex); - final int numShards = sourceIndexService != null ? sourceIndexService.getIndexSettings().getNumberOfShards() : -1; + final Set requiredShards = IndexMetaData.selectRecoverFromShards(shardId().id(), + sourceIndexService.getMetaData(), indexMetaData.getNumberOfShards()); + final int numShards = sourceIndexService != null ? requiredShards.size() : -1; if (sourceIndexService != null) { for (IndexShard shard : sourceIndexService) { - if (shard.state() == IndexShardState.STARTED) { + if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) { startedShards.add(shard); } } @@ -2041,10 +2043,8 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService markAsRecovering("from local shards", recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(() -> { try { - final Set shards = IndexMetaData.selectShrinkShards(shardId().id(), sourceIndexService.getMetaData(), - +indexMetaData.getNumberOfShards()); if (recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream() - .filter((s) -> shards.contains(s.shardId())).collect(Collectors.toList()))) { + .filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()))) { recoveryListener.onRecoveryDone(recoveryState); } } catch (Exception e) { diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java new file mode 100644 index 0000000000000..15f050a80173b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -0,0 +1,249 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BitSetIterator; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.Uid; + +import java.io.IOException; +import java.util.function.IntConsumer; +import java.util.function.Predicate; + +/** + * A query that selects all docs that do NOT belong in the current shards this query is executed on. + * It can be used to split a shard into N shards marking every document that doesn't belong into the shard + * as deleted. See {@link org.apache.lucene.index.IndexWriter#deleteDocuments(Query...)} + */ +final class ShardSplittingQuery extends Query { + private final IndexMetaData indexMetaData; + private final int shardId; + + ShardSplittingQuery(IndexMetaData indexMetaData, int shardId) { + this.indexMetaData = indexMetaData; + this.shardId = shardId; + } + + @Override + public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) { + return new ConstantScoreWeight(this, boost) { + @Override + public String toString() { + return "weight(delete docs query)"; + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + LeafReader leafReader = context.reader(); + FixedBitSet bitSet = new FixedBitSet(leafReader.maxDoc()); + Terms terms = leafReader.terms(RoutingFieldMapper.NAME); + Predicate includeInShard = ref -> { + int targetShardId = OperationRouting.generateShardId(indexMetaData, + Uid.decodeId(ref.bytes, ref.offset, ref.length), null); + return shardId == targetShardId; + }; + if (terms == null) { // this is the common case - no partitioning and no _routing values + findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, bitSet::set); + } else { + if (indexMetaData.isRoutingPartitionedIndex()) { + // this is the heaviest invariant. Here we have to visit all docs stored fields do extract _id and _routing + // this this index is routing partitioned. + Bits liveDocs = leafReader.getLiveDocs(); + Visitor visitor = new Visitor(); + return new ConstantScoreScorer(this, score(), + new RoutingPartitionedDocIdSetIterator(leafReader, liveDocs, visitor)); + } else { + // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to delete + findSplitDocs(RoutingFieldMapper.NAME, ref -> { + int targetShardId = OperationRouting.generateShardId(indexMetaData, null, ref.utf8ToString()); + return shardId == targetShardId; + }, leafReader, bitSet::set); + // now if we have a mixed index where some docs have a _routing value and some don't we have to exclude the ones + // with a routing value from the next iteration an delete / select based on the ID. + if (terms.getDocCount() != leafReader.maxDoc()) { + // this is a special case where some of the docs have no routing values this sucks but it's possible today + FixedBitSet hasRoutingValue = new FixedBitSet(leafReader.maxDoc()); + findSplitDocs(RoutingFieldMapper.NAME, ref -> false, leafReader, + hasRoutingValue::set); + findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, docId -> { + if (hasRoutingValue.get(docId) == false) { + bitSet.set(docId); + } + }); + } + } + } + return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length())); + } + }; + } + + @Override + public String toString(String field) { + return "shard_splitting_query"; + } + + @Override + public boolean equals(Object o) { + return sameClassAs(o); + } + + @Override + public int hashCode() { + return classHash(); + } + + private static void findSplitDocs(String idField, Predicate includeInShard, + LeafReader leafReader, IntConsumer consumer) throws IOException { + Terms terms = leafReader.terms(idField); + TermsEnum iterator = terms.iterator(); + BytesRef idTerm; + PostingsEnum postingsEnum = null; + while ((idTerm = iterator.next()) != null) { + if (includeInShard.test(idTerm) == false) { + postingsEnum = iterator.postings(postingsEnum); + int doc; + while ((doc = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + consumer.accept(doc); + } + } + } + } + + private static final class Visitor extends StoredFieldVisitor { + int leftToVisit = 2; + final BytesRef spare = new BytesRef(); + String routing; + String id; + + void reset() { + routing = id = null; + leftToVisit = 2; + } + + @Override + public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException { + switch (fieldInfo.name) { + case IdFieldMapper.NAME: + id = Uid.decodeId(value); + break; + default: + throw new IllegalStateException("Unexpected field: " + fieldInfo.name); + } + } + + @Override + public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { + spare.bytes = value; + spare.offset = 0; + spare.length = value.length; + switch (fieldInfo.name) { + case RoutingFieldMapper.NAME: + routing = spare.utf8ToString(); + break; + default: + throw new IllegalStateException("Unexpected field: " + fieldInfo.name); + } + } + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + switch (fieldInfo.name) { + case IdFieldMapper.NAME: + case RoutingFieldMapper.NAME: + leftToVisit--; + return Status.YES; + default: + return leftToVisit == 0 ? Status.STOP : Status.NO; + + } + } + } + + /** + * This DISI visits every live doc and selects all docs that don't belong into this + * shard based on their id and rounting value. This is only used in a routing partitioned index. + */ + private final class RoutingPartitionedDocIdSetIterator extends DocIdSetIterator { + private final LeafReader leafReader; + private final Bits liveDocs; + private final Visitor visitor; + private int doc; + + RoutingPartitionedDocIdSetIterator(LeafReader leafReader, Bits liveDocs, Visitor visitor) { + this.leafReader = leafReader; + this.liveDocs = liveDocs; + this.visitor = visitor; + doc = -1; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int nextDoc() throws IOException { + while (++doc < leafReader.maxDoc()) { + if (liveDocs == null || liveDocs.get(doc)) { + visitor.reset(); + leafReader.document(doc, visitor); + int targetShardId = OperationRouting.generateShardId(indexMetaData, visitor.id, visitor.routing); + if (targetShardId != shardId) { // move to next doc if we can keep it + return doc; + } + } + } + return doc = DocIdSetIterator.NO_MORE_DOCS; + } + + @Override + public int advance(int target) throws IOException { + while (nextDoc() < target) {} + return doc; + } + + @Override + public long cost() { + return leafReader.maxDoc(); + } + } +} + + diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index e5053fc7882e0..cfd9972c55a05 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -31,6 +31,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.cluster.routing.RecoverySource; @@ -114,6 +115,9 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); // now that the mapping is merged we can validate the index sort configuration. Sort indexSort = indexShard.getIndexSort(); + final boolean isSplit = indexMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); + assert isSplit == false || indexMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + + "single type but the index is created before 6.0.0"; return executeRecovery(indexShard, () -> { logger.debug("starting recovery from local shards {}", shards); try { @@ -122,7 +126,8 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate final long maxSeqNo = shards.stream().mapToLong(LocalShardSnapshot::maxSeqNo).max().getAsLong(); final long maxUnsafeAutoIdTimestamp = shards.stream().mapToLong(LocalShardSnapshot::maxUnsafeAutoIdTimestamp).max().getAsLong(); - addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp); + addIndices(indexShard.recoveryState().getIndex(), directory, indexSort, sources, maxSeqNo, maxUnsafeAutoIdTimestamp, + indexShard.indexSettings().getIndexMetaData(), indexShard.shardId().id(), isSplit); internalRecoverFromStore(indexShard); // just trigger a merge to do housekeeping on the // copied segments - we will also see them in stats etc. @@ -136,13 +141,9 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate return false; } - void addIndices( - final RecoveryState.Index indexRecoveryStats, - final Directory target, - final Sort indexSort, - final Directory[] sources, - final long maxSeqNo, - final long maxUnsafeAutoIdTimestamp) throws IOException { + void addIndices(final RecoveryState.Index indexRecoveryStats, final Directory target, final Sort indexSort, final Directory[] sources, + final long maxSeqNo, final long maxUnsafeAutoIdTimestamp, IndexMetaData indexMetaData, int shardId, boolean split) + throws IOException { final Directory hardLinkOrCopyTarget = new org.apache.lucene.store.HardlinkCopyDirectoryWrapper(target); IndexWriterConfig iwc = new IndexWriterConfig(null) .setCommitOnClose(false) @@ -154,8 +155,13 @@ void addIndices( if (indexSort != null) { iwc.setIndexSort(indexSort); } + try (IndexWriter writer = new IndexWriter(new StatsDirectoryWrapper(hardLinkOrCopyTarget, indexRecoveryStats), iwc)) { writer.addIndexes(sources); + + if (split) { + writer.deleteDocuments(new ShardSplittingQuery(indexMetaData, shardId)); + } /* * We set the maximum sequence number and the local checkpoint on the target to the maximum of the maximum sequence numbers on * the source shards. This ensures that history after this maximum sequence number can advance and we have correct @@ -272,7 +278,7 @@ private boolean canRecover(IndexShard indexShard) { // got closed on us, just ignore this recovery return false; } - if (!indexShard.routingEntry().primary()) { + if (indexShard.routingEntry().primary() == false) { throw new IndexShardRecoveryException(shardId, "Trying to recover when the shard is in backup state", null); } return true; diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java index 10b46be6760bb..a0071d70758af 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestShrinkIndexAction.java @@ -19,8 +19,9 @@ package org.elasticsearch.rest.action.admin.indices; -import org.elasticsearch.action.admin.indices.shrink.ShrinkRequest; -import org.elasticsearch.action.admin.indices.shrink.ShrinkResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.settings.Settings; @@ -52,14 +53,15 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (request.param("index") == null) { throw new IllegalArgumentException("no source index"); } - ShrinkRequest shrinkIndexRequest = new ShrinkRequest(request.param("target"), request.param("index")); - request.applyContentParser(parser -> ShrinkRequest.PARSER.parse(parser, shrinkIndexRequest, null)); + ResizeRequest shrinkIndexRequest = new ResizeRequest(request.param("target"), request.param("index")); + shrinkIndexRequest.setResizeType(ResizeType.SHRINK); + request.applyContentParser(parser -> ResizeRequest.PARSER.parse(parser, shrinkIndexRequest, null)); shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout())); shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout())); shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); - return channel -> client.admin().indices().shrinkIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { + return channel -> client.admin().indices().resizeIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { @Override - public void addCustomFields(XContentBuilder builder, ShrinkResponse response) throws IOException { + public void addCustomFields(XContentBuilder builder, ResizeResponse response) throws IOException { response.addCustomFields(builder); } }); diff --git a/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java new file mode 100644 index 0000000000000..dcc811bd0177b --- /dev/null +++ b/core/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSplitIndexAction.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action.admin.indices; + +import org.elasticsearch.action.admin.indices.shrink.ResizeRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.support.ActiveShardCount; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.AcknowledgedRestListener; + +import java.io.IOException; + +public class RestSplitIndexAction extends BaseRestHandler { + public RestSplitIndexAction(Settings settings, RestController controller) { + super(settings); + controller.registerHandler(RestRequest.Method.PUT, "/{index}/_split/{target}", this); + controller.registerHandler(RestRequest.Method.POST, "/{index}/_split/{target}", this); + } + + @Override + public String getName() { + return "split_index_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + if (request.param("target") == null) { + throw new IllegalArgumentException("no target index"); + } + if (request.param("index") == null) { + throw new IllegalArgumentException("no source index"); + } + ResizeRequest shrinkIndexRequest = new ResizeRequest(request.param("target"), request.param("index")); + shrinkIndexRequest.setResizeType(ResizeType.SPLIT); + request.applyContentParser(parser -> ResizeRequest.PARSER.parse(parser, shrinkIndexRequest, null)); + shrinkIndexRequest.timeout(request.paramAsTime("timeout", shrinkIndexRequest.timeout())); + shrinkIndexRequest.masterNodeTimeout(request.paramAsTime("master_timeout", shrinkIndexRequest.masterNodeTimeout())); + shrinkIndexRequest.setWaitForActiveShards(ActiveShardCount.parseString(request.param("wait_for_active_shards"))); + return channel -> client.admin().indices().resizeIndex(shrinkIndexRequest, new AcknowledgedRestListener(channel) { + @Override + public void addCustomFields(XContentBuilder builder, ResizeResponse response) throws IOException { + response.addCustomFields(builder); + } + }); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java index 3c2e10d181b58..982b9456b8cdf 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/ShrinkIndexIT.java @@ -105,7 +105,7 @@ public void testCreateShrinkIndexToN() { .put("index.blocks.write", true)).get(); ensureGreen(); // now merge source into a 4 shard index - assertAcked(client().admin().indices().prepareShrinkIndex("source", "first_shrink") + assertAcked(client().admin().indices().prepareResizeIndex("source", "first_shrink") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", shardSplits[1]).build()).get()); @@ -127,7 +127,7 @@ public void testCreateShrinkIndexToN() { .put("index.blocks.write", true)).get(); ensureGreen(); // now merge source into a 2 shard index - assertAcked(client().admin().indices().prepareShrinkIndex("first_shrink", "second_shrink") + assertAcked(client().admin().indices().prepareResizeIndex("first_shrink", "second_shrink") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", shardSplits[2]).build()).get()); @@ -211,7 +211,7 @@ public void testShrinkIndexPrimaryTerm() throws Exception { // now merge source into target final Settings shrinkSettings = Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", numberOfTargetShards).build(); - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target").setSettings(shrinkSettings).get()); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target").setSettings(shrinkSettings).get()); ensureGreen(); @@ -264,7 +264,7 @@ public void testCreateShrinkIndex() { // now merge source into a single shard index final boolean createWithReplicas = randomBoolean(); - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target") + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder().put("index.number_of_replicas", createWithReplicas ? 1 : 0).build()).get()); ensureGreen(); @@ -350,7 +350,7 @@ public void testCreateShrinkIndexFails() throws Exception { ensureGreen(); // now merge source into a single shard index - client().admin().indices().prepareShrinkIndex("source", "target") + client().admin().indices().prepareResizeIndex("source", "target") .setWaitForActiveShards(ActiveShardCount.NONE) .setSettings(Settings.builder() .put("index.routing.allocation.exclude._name", mergeNode) // we manually exclude the merge node to forcefully fuck it up @@ -436,16 +436,16 @@ public void testCreateShrinkWithIndexSort() throws Exception { // check that index sort cannot be set on the target index IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, - () -> client().admin().indices().prepareShrinkIndex("source", "target") + () -> client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", "2") .put("index.sort.field", "foo") .build()).get()); - assertThat(exc.getMessage(), containsString("can't override index sort when shrinking index")); + assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); // check that the index sort order of `source` is correctly applied to the `target` - assertAcked(client().admin().indices().prepareShrinkIndex("source", "target") + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") .setSettings(Settings.builder() .put("index.number_of_replicas", 0) .put("index.number_of_shards", "2").build()).get()); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java new file mode 100644 index 0000000000000..76d5140a9bfdb --- /dev/null +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -0,0 +1,452 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.indices.create; + +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedSetSelector; +import org.apache.lucene.search.SortedSetSortField; +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest; +import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse; +import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; +import org.elasticsearch.action.admin.indices.stats.CommonStats; +import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; +import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.routing.Murmur3HashFunction; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.index.seqno.SeqNoStats; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.VersionUtils; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.stream.IntStream; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + + +public class SplitIndexIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList(InternalSettingsPlugin.class); + } + + public void testCreateSplitIndexToN() { + int[][] possibleShardSplits = new int[][] {{2,4,8}, {3, 6, 12}, {1, 2, 4}}; + int[] shardSplits = randomFrom(possibleShardSplits); + assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + internalCluster().ensureAtLeastNumDataNodes(2); + final boolean useRouting = randomBoolean(); + final boolean useMixedRouting = useRouting ? randomBoolean() : false; + CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); + Settings.Builder settings = Settings.builder().put(indexSettings()) + .put("number_of_shards", shardSplits[0]) + .put("index.routing_shards_factor", shardSplits[2] / shardSplits[0]); + if (useRouting && useMixedRouting == false && randomBoolean()) { + settings.put("index.routing_partition_size", randomIntBetween(1, 10)); + createInitialIndex.addMapping("t1", "_routing", "required=true"); + } + logger.info("use routing {} use mixed routing {}", useRouting, useMixedRouting); + createInitialIndex.setSettings(settings).get(); + + int numDocs = randomIntBetween(10, 50); + String[] routingValue = new String[numDocs]; + for (int i = 0; i < numDocs; i++) { + IndexRequestBuilder builder = client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); + if (useMixedRouting && randomBoolean()) { + routingValue[i] = routing; + + } else { + routingValue[i] = routing; + } + builder.setRouting(routingValue[i]); + } + builder.get(); + } + + if (randomBoolean()) { + for (int i = 0; i < numDocs; i++) { // let's introduce some updates / deletes on the index + if (randomBoolean()) { + IndexRequestBuilder builder = client().prepareIndex("source", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + } + } + + ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() + .getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class); + // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node + // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due + // to the require._name below. + ensureYellow(); + // relocate all shards to one node such that we can merge it. + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "first_split") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", shardSplits[1]).build()).get()); + ensureGreen(); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + for (int i = 0; i < numDocs; i++) { // now update + IndexRequestBuilder builder = client().prepareIndex("first_split", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + flushAndRefresh(); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + for (int i = 0; i < numDocs; i++) { + GetResponse getResponse = client().prepareGet("first_split", "t1", Integer.toString(i)).setRouting(routingValue[i]).get(); + assertTrue(getResponse.isExists()); + } + + // relocate all shards to one node such that we can merge it. + client().admin().indices().prepareUpdateSettings("first_split") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + // now merge source into a 2 shard index + assertAcked(client().admin().indices().prepareResizeIndex("first_split", "second_split") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", shardSplits[2]).build()).get()); + ensureGreen(); + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + // let it be allocated anywhere and bump replicas + client().admin().indices().prepareUpdateSettings("second_split") + .setSettings(Settings.builder() + .put("index.number_of_replicas", 1)).get(); + ensureGreen(); + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + for (int i = 0; i < numDocs; i++) { // now update + IndexRequestBuilder builder = client().prepareIndex("second_split", "t1", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON); + if (useRouting) { + builder.setRouting(routingValue[i]); + } + builder.get(); + } + flushAndRefresh(); + for (int i = 0; i < numDocs; i++) { + GetResponse getResponse = client().prepareGet("second_split", "t1", Integer.toString(i)).setRouting(routingValue[i]).get(); + assertTrue(getResponse.isExists()); + } + assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + } + + public void testSplitIndexPrimaryTerm() throws Exception { + final List factors = Arrays.asList(1, 2, 4, 8); + final List numberOfShardsFactors = randomSubsetOf(scaledRandomIntBetween(1, factors.size()), factors); + final int numberOfShards = randomSubsetOf(numberOfShardsFactors).stream().reduce(1, (x, y) -> x * y); + final int numberOfTargetShards = numberOfShardsFactors.stream().reduce(2, (x, y) -> x * y); + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) + .put("number_of_shards", numberOfShards) + .put("index.routing_shards_factor", numberOfTargetShards / numberOfShards)).get(); + + final ImmutableOpenMap dataNodes = + client().admin().cluster().prepareState().get().getState().nodes().getDataNodes(); + assertThat(dataNodes.size(), greaterThanOrEqualTo(2)); + ensureYellow(); + + // fail random primary shards to force primary terms to increase + final Index source = resolveIndex("source"); + final int iterations = scaledRandomIntBetween(0, 16); + for (int i = 0; i < iterations; i++) { + final String node = randomSubsetOf(1, internalCluster().nodesInclude("source")).get(0); + final IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node); + final IndexService indexShards = indexServices.indexServiceSafe(source); + for (final Integer shardId : indexShards.shardIds()) { + final IndexShard shard = indexShards.getShard(shardId); + if (shard.routingEntry().primary() && randomBoolean()) { + disableAllocation("source"); + shard.failShard("test", new Exception("test")); + // this can not succeed until the shard is failed and a replica is promoted + int id = 0; + while (true) { + // find an ID that routes to the right shard, we will only index to the shard that saw a primary failure + final String s = Integer.toString(id); + final int hash = Math.floorMod(Murmur3HashFunction.hash(s), numberOfShards); + if (hash == shardId) { + final IndexRequest request = + new IndexRequest("source", "type", s).source("{ \"f\": \"" + s + "\"}", XContentType.JSON); + client().index(request).get(); + break; + } else { + id++; + } + } + enableAllocation("source"); + ensureGreen(); + } + } + } + + final Settings.Builder prepareSplitSettings = Settings.builder().put("index.blocks.write", true); + client().admin().indices().prepareUpdateSettings("source").setSettings(prepareSplitSettings).get(); + ensureYellow(); + + final IndexMetaData indexMetaData = indexMetaData(client(), "source"); + final long beforeSplitPrimaryTerm = IntStream.range(0, numberOfShards).mapToLong(indexMetaData::primaryTerm).max().getAsLong(); + + // now split source into target + final Settings splitSettings = + Settings.builder().put("index.number_of_replicas", 0).put("index.number_of_shards", numberOfTargetShards).build(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(splitSettings).get()); + + ensureGreen(); + + final IndexMetaData aftersplitIndexMetaData = indexMetaData(client(), "target"); + for (int shardId = 0; shardId < numberOfTargetShards; shardId++) { + assertThat(aftersplitIndexMetaData.primaryTerm(shardId), equalTo(beforeSplitPrimaryTerm + 1)); + } + } + + private static IndexMetaData indexMetaData(final Client client, final String index) { + final ClusterStateResponse clusterStateResponse = client.admin().cluster().state(new ClusterStateRequest()).actionGet(); + return clusterStateResponse.getState().metaData().index(index); + } + + public void testCreateSplitIndex() { + internalCluster().ensureAtLeastNumDataNodes(2); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_alpha1, Version.CURRENT); + prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) + .put("number_of_shards", 1) + .put("index.version.created", version) + .put("index.routing_shards_factor", 2) + ).get(); + final int docs = randomIntBetween(0, 128); + for (int i = 0; i < docs; i++) { + client().prepareIndex("source", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + ImmutableOpenMap dataNodes = + client().admin().cluster().prepareState().get().getState().nodes().getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node + // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due + // to the require._name below. + ensureGreen(); + // relocate all shards to one node such that we can merge it. + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureGreen(); + + final IndicesStatsResponse sourceStats = client().admin().indices().prepareStats("source").setSegments(true).get(); + + // disable rebalancing to be able to capture the right stats. balancing can move the target primary + // making it hard to pin point the source shards. + client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put( + EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none" + )).get(); + try { + + // now merge source into a single shard index + final boolean createWithReplicas = randomBoolean(); + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", createWithReplicas ? 1 : 0) + .put("index.number_of_shards", 2).build()).get()); + ensureGreen(); + + // resolve true merge node - this is not always the node we required as all shards may be on another node + final ClusterState state = client().admin().cluster().prepareState().get().getState(); + DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId()); + logger.info("split node {}", mergeNode); + + final long maxSeqNo = Arrays.stream(sourceStats.getShards()) + .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())) + .map(ShardStats::getSeqNoStats).mapToLong(SeqNoStats::getMaxSeqNo).max().getAsLong(); + final long maxUnsafeAutoIdTimestamp = Arrays.stream(sourceStats.getShards()) + .filter(shard -> shard.getShardRouting().currentNodeId().equals(mergeNode.getId())) + .map(ShardStats::getStats) + .map(CommonStats::getSegments) + .mapToLong(SegmentsStats::getMaxUnsafeAutoIdTimestamp) + .max() + .getAsLong(); + + final IndicesStatsResponse targetStats = client().admin().indices().prepareStats("target").get(); + for (final ShardStats shardStats : targetStats.getShards()) { + final SeqNoStats seqNoStats = shardStats.getSeqNoStats(); + final ShardRouting shardRouting = shardStats.getShardRouting(); + assertThat("failed on " + shardRouting, seqNoStats.getMaxSeqNo(), equalTo(maxSeqNo)); + assertThat("failed on " + shardRouting, seqNoStats.getLocalCheckpoint(), equalTo(maxSeqNo)); + assertThat("failed on " + shardRouting, + shardStats.getStats().getSegments().getMaxUnsafeAutoIdTimestamp(), equalTo(maxUnsafeAutoIdTimestamp)); + } + + final int size = docs > 0 ? 2 * docs : 1; + assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + + if (createWithReplicas == false) { + // bump replicas + client().admin().indices().prepareUpdateSettings("target") + .setSettings(Settings.builder() + .put("index.number_of_replicas", 1)).get(); + ensureGreen(); + assertHitCount(client().prepareSearch("target").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + } + + for (int i = docs; i < 2 * docs; i++) { + client().prepareIndex("target", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + flushAndRefresh(); + assertHitCount(client().prepareSearch("target").setSize(2 * size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), + 2 * docs); + assertHitCount(client().prepareSearch("source").setSize(size).setQuery(new TermsQueryBuilder("foo", "bar")).get(), docs); + GetSettingsResponse target = client().admin().indices().prepareGetSettings("target").get(); + assertEquals(version, target.getIndexToSettings().get("target").getAsVersion("index.version.created", null)); + } finally { + // clean up + client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put( + EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String)null + )).get(); + } + + } + + public void testCreateSplitWithIndexSort() throws Exception { + SortField expectedSortField = new SortedSetSortField("id", true, SortedSetSelector.Type.MAX); + expectedSortField.setMissingValue(SortedSetSortField.STRING_FIRST); + Sort expectedIndexSort = new Sort(expectedSortField); + internalCluster().ensureAtLeastNumDataNodes(2); + prepareCreate("source") + .setSettings( + Settings.builder() + .put(indexSettings()) + .put("sort.field", "id") + .put("index.routing_shards_factor", 16) + .put("sort.order", "desc") + .put("number_of_shards", 2) + .put("number_of_replicas", 0) + ) + .addMapping("type", "id", "type=keyword,doc_values=true") + .get(); + for (int i = 0; i < 20; i++) { + client().prepareIndex("source", "type", Integer.toString(i)) + .setSource("{\"foo\" : \"bar\", \"id\" : " + i + "}", XContentType.JSON).get(); + } + ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() + .getDataNodes(); + assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); + DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class); + String mergeNode = discoveryNodes[0].getName(); + // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node + // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due + // to the require._name below. + ensureGreen(); + + flushAndRefresh(); + assertSortedSegments("source", expectedIndexSort); + + // relocate all shards to one node such that we can merge it. + client().admin().indices().prepareUpdateSettings("source") + .setSettings(Settings.builder() + .put("index.blocks.write", true)).get(); + ensureYellow(); + + // check that index sort cannot be set on the target index + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, + () -> client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 4) + .put("index.sort.field", "foo") + .build()).get()); + assertThat(exc.getMessage(), containsString("can't override index sort when resizing an index")); + + // check that the index sort order of `source` is correctly applied to the `target` + assertAcked(client().admin().indices().prepareResizeIndex("source", "target") + .setResizeType(ResizeType.SPLIT) + .setSettings(Settings.builder() + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 4).build()).get()); + ensureGreen(); + flushAndRefresh(); + GetSettingsResponse settingsResponse = + client().admin().indices().prepareGetSettings("target").execute().actionGet(); + assertEquals(settingsResponse.getSetting("target", "index.sort.field"), "id"); + assertEquals(settingsResponse.getSetting("target", "index.sort.order"), "desc"); + assertSortedSegments("target", expectedIndexSort); + + // ... and that the index sort is also applied to updates + for (int i = 20; i < 40; i++) { + client().prepareIndex("target", "type") + .setSource("{\"foo\" : \"bar\", \"i\" : " + i + "}", XContentType.JSON).get(); + } + flushAndRefresh(); + assertSortedSegments("target", expectedIndexSort); + } +} diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java similarity index 92% rename from core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java rename to core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index b24c8dca79a58..fd6b73bfc2cad 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -49,7 +49,7 @@ import static java.util.Collections.emptyMap; -public class TransportShrinkActionTests extends ESTestCase { +public class TransportResizeActionTests extends ESTestCase { private ClusterState createClusterState(String name, int numShards, int numReplicas, Settings settings) { MetaData.Builder metaBuilder = MetaData.builder(); @@ -72,18 +72,18 @@ public void testErrorCondition() { Settings.builder().put("index.blocks.write", true).build()); assertTrue( expectThrows(IllegalStateException.class, () -> - TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), state, + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)) ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); assertTrue( expectThrows(IllegalStateException.class, () -> { - ShrinkRequest req = new ShrinkRequest("target", "source"); - req.getShrinkIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); + ResizeRequest req = new ResizeRequest("target", "source"); + req.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); - TransportShrinkAction.prepareCreateIndexRequest(req, clusterState, + TransportResizeAction.prepareCreateIndexRequest(req, clusterState, (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, new IndexNameExpressionResolver(Settings.EMPTY)); } @@ -105,7 +105,7 @@ public void testErrorCondition() { routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - TransportShrinkAction.prepareCreateIndexRequest(new ShrinkRequest("target", "source"), clusterState, + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)); } @@ -129,14 +129,14 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int numSourceShards = clusterState.metaData().index(indexName).getNumberOfShards(); DocsStats stats = new DocsStats(randomIntBetween(0, (IndexWriter.MAX_DOCS) / numSourceShards), randomIntBetween(1, 1000)); - ShrinkRequest target = new ShrinkRequest("target", indexName); + ResizeRequest target = new ResizeRequest("target", indexName); final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); - CreateIndexClusterStateUpdateRequest request = TransportShrinkAction.prepareCreateIndexRequest( + CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( target, clusterState, (i) -> stats, new IndexNameExpressionResolver(Settings.EMPTY)); - assertNotNull(request.shrinkFrom()); - assertEquals(indexName, request.shrinkFrom().getName()); + assertNotNull(request.recoverFrom()); + assertEquals(indexName, request.recoverFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); assertEquals("shrink_index", request.cause()); assertEquals(request.waitForActiveShards(), activeShardCount); diff --git a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java index 81acd138d26fb..6fd3d66c8f81b 100644 --- a/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/ClusterModuleTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.RebalanceOnlyWhenActiveAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ReplicaAfterPrimaryActiveAllocationDecider; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SameShardAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider; @@ -174,6 +175,7 @@ public void testShardsAllocatorFactoryNull() { public void testAllocationDeciderOrder() { List> expectedDeciders = Arrays.asList( MaxRetryAllocationDecider.class, + ResizeAllocationDecider.class, ReplicaAfterPrimaryActiveAllocationDecider.class, RebalanceOnlyWhenActiveAllocationDecider.class, ClusterRebalanceAllocationDecider.class, diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java index 4dd757c140311..f44d0b7c4036e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexCreationTaskTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.block.ClusterBlock; @@ -258,8 +259,8 @@ public void testIndexRemovalOnFailure() throws Exception { public void testShrinkIndexIgnoresTemplates() throws Exception { final Index source = new Index("source_idx", "aaa111bbb222"); - when(request.shrinkFrom()).thenReturn(source); - + when(request.recoverFrom()).thenReturn(source); + when(request.resizeType()).thenReturn(ResizeType.SHRINK); currentStateMetaDataBuilder.put(createIndexMetaDataBuilder("source_idx", "aaa111bbb222", 2, 2)); routingTableBuilder.add(createIndexRoutingTableWithStartedShards(source)); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index fa56c756fcc35..85ba918068eba 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Collections; import java.util.Set; import static org.hamcrest.Matchers.is; @@ -84,21 +85,12 @@ public void testIndexMetaDataSerialization() throws IOException { } public void testGetRoutingFactor() { - int numberOfReplicas = randomIntBetween(0, 10); - IndexMetaData metaData = IndexMetaData.builder("foo") - .settings(Settings.builder() - .put("index.version.created", 1) - .put("index.number_of_shards", 32) - .put("index.number_of_replicas", numberOfReplicas) - .build()) - .creationDate(randomLong()) - .build(); Integer numShard = randomFrom(1, 2, 4, 8, 16); - int routingFactor = IndexMetaData.getRoutingFactor(metaData, numShard); - assertEquals(routingFactor * numShard, metaData.getNumberOfShards()); + int routingFactor = IndexMetaData.getRoutingFactor(32, numShard); + assertEquals(routingFactor * numShard, 32); - Integer brokenNumShards = randomFrom(3, 5, 9, 12, 29, 42, 64); - expectThrows(IllegalArgumentException.class, () -> IndexMetaData.getRoutingFactor(metaData, brokenNumShards)); + Integer brokenNumShards = randomFrom(3, 5, 9, 12, 29, 42); + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.getRoutingFactor(32, brokenNumShards)); } public void testSelectShrinkShards() { @@ -125,6 +117,60 @@ public void testSelectShrinkShards() { expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectShrinkShards(8, metaData, 8)).getMessage()); } + public void testSelectResizeShards() { + IndexMetaData split = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .build(); + + IndexMetaData shrink = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 32) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .build(); + int numTargetShards = randomFrom(4, 6, 8, 12); + int shard = randomIntBetween(0, numTargetShards-1); + assertEquals(Collections.singleton(IndexMetaData.selectSplitShard(shard, split, numTargetShards)), + IndexMetaData.selectRecoverFromShards(shard, split, numTargetShards)); + + numTargetShards = randomFrom(1, 2, 4, 8, 16); + shard = randomIntBetween(0, numTargetShards-1); + assertEquals(IndexMetaData.selectShrinkShards(shard, shrink, numTargetShards), + IndexMetaData.selectRecoverFromShards(shard, shrink, numTargetShards)); + + assertEquals("can't select recover from shards if both indices have the same number of shards", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectRecoverFromShards(0, shrink, 32)).getMessage()); + } + + public void testSelectSplitShard() { + IndexMetaData metaData = IndexMetaData.builder("foo") + .settings(Settings.builder() + .put("index.version.created", 1) + .put("index.number_of_shards", 2) + .put("index.number_of_replicas", 0) + .build()) + .creationDate(randomLong()) + .build(); + ShardId shardId = IndexMetaData.selectSplitShard(0, metaData, 4); + assertEquals(0, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(1, metaData, 4); + assertEquals(0, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(2, metaData, 4); + assertEquals(1, shardId.getId()); + shardId = IndexMetaData.selectSplitShard(3, metaData, 4); + assertEquals(1, shardId.getId()); + + assertEquals("the number of target shards (0) must be greater than the shard id: 0", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 0)).getMessage()); + } + public void testIndexFormat() { Settings defaultSettings = Settings.builder() .put("index.version.created", 1) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 7bfc7872f816a..34c45a8cd2414 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.shrink.ResizeType; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.EmptyClusterInfoService; @@ -43,6 +44,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; import java.util.List; @@ -75,6 +77,12 @@ public static boolean isShrinkable(int source, int target) { return target * x == source; } + public static boolean isSplitable(int source, int target) { + int x = target / source; + assert source < target : source + " >= " + target; + return source * x == target; + } + public void testValidateShrinkIndex() { int numShards = randomIntBetween(2, 42); ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), @@ -103,7 +111,7 @@ public void testValidateShrinkIndex() { ).getMessage()); - assertEquals("index source must be read-only to shrink index. use \"index.blocks.write=true\"", + assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateShrinkIndex( createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) @@ -122,7 +130,7 @@ public void testValidateShrinkIndex() { Settings.builder().put("index.number_of_shards", 3).build()) ).getMessage()); - assertEquals("mappings are not allowed when shrinking indices, all mappings are copied from the source index", + assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", expectThrows(IllegalArgumentException.class, () -> { MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.singleton("foo"), "target", Settings.EMPTY); @@ -151,11 +159,77 @@ public void testValidateShrinkIndex() { Settings.builder().put("index.number_of_shards", targetShards).build()); } - public void testShrinkIndexSettings() { + public void testValidateSplitIndex() { + int numShards = randomIntBetween(1, 42); + ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), + Settings.builder().put("index.blocks.write", true).build()); + + assertEquals("index [source] already exists", + expectThrows(ResourceAlreadyExistsException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(state, "target", Collections.emptySet(), "source", Settings.EMPTY) + ).getMessage()); + + assertEquals("no such index", + expectThrows(IndexNotFoundException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY) + ).getMessage()); + + assertEquals("the number of source shards must be less that the number of target shards", + expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", + 10, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), + "target", Settings.builder().put("index.number_of_shards", 5).build()) + ).getMessage()); + + + assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", + expectThrows(IllegalStateException.class, () -> + MetaDataCreateIndexService.validateSplitIndex( + createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) + , "source", Collections.emptySet(), "target", Settings.EMPTY) + ).getMessage()); + + + assertEquals("the number of source shards [4] must be a must be a multiple of [3]", + expectThrows(IllegalArgumentException.class, () -> + MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", 3, randomIntBetween(0, 10), + Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", + Settings.builder().put("index.number_of_shards", 4).build()) + ).getMessage()); + + assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", + expectThrows(IllegalArgumentException.class, () -> { + MetaDataCreateIndexService.validateSplitIndex(state, "source", Collections.singleton("foo"), + "target", Settings.EMPTY); + } + ).getMessage()); + + + ClusterState clusterState = ClusterState.builder(createClusterState("source", numShards, 0, + Settings.builder().put("index.blocks.write", true).build())).nodes(DiscoveryNodes.builder().add(newNode("node1"))) + .build(); + AllocationService service = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new MaxRetryAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + + RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + // now we start the shard + routingTable = service.applyStartedShards(clusterState, + routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + int targetShards; + do { + targetShards = randomIntBetween(numShards, 100); + } while (isSplitable(numShards, targetShards) == false); + MetaDataCreateIndexService.validateSplitIndex(clusterState, "source", Collections.emptySet(), "target", + Settings.builder().put("index.number_of_shards", targetShards).build()); + } + + public void testResizeIndexSettings() { String indexName = randomAlphaOfLength(10); List versions = Arrays.asList(VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random()), VersionUtils.randomVersion(random())); - versions.sort((l, r) -> Long.compare(l.id, r.id)); + versions.sort(Comparator.comparingLong(l -> l.id)); Version version = versions.get(0); Version minCompat = versions.get(1); Version upgraded = versions.get(2); @@ -182,8 +256,8 @@ public void testShrinkIndexSettings() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); Settings.Builder builder = Settings.builder(); - MetaDataCreateIndexService.prepareShrinkIndexSettings( - clusterState, Collections.emptySet(), builder, clusterState.metaData().index(indexName).getIndex(), "target"); + MetaDataCreateIndexService.prepareResizeIndexSettings(clusterState, Collections.emptySet(), builder, + clusterState.metaData().index(indexName).getIndex(), "target", ResizeType.SHRINK); assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type")); assertEquals("analysis settings must be copied", "keyword", builder.build().get("index.analysis.analyzer.my_analyzer.tokenizer")); diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java index 498edee12f90a..1f8de1ca02fd7 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/OperationRoutingTests.java @@ -23,9 +23,7 @@ import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -54,6 +52,7 @@ public class OperationRoutingTests extends ESTestCase{ + public void testGenerateShardId() { int[][] possibleValues = new int[][] { {8,4,2}, {20, 10, 2}, {36, 12, 3}, {15,5,1} @@ -70,6 +69,7 @@ public void testGenerateShardId() { .numberOfReplicas(1) .setRoutingNumShards(shardSplits[0]).build(); int shrunkShard = OperationRouting.generateShardId(shrunk, term, null); + Set shardIds = IndexMetaData.selectShrinkShards(shrunkShard, metaData, shrunk.getNumberOfShards()); assertEquals(1, shardIds.stream().filter((sid) -> sid.id() == shard).count()); @@ -81,6 +81,36 @@ public void testGenerateShardId() { } } + public void testGenerateShardIdSplit() { + int[][] possibleValues = new int[][] { + {2,4,8}, {2, 10, 20}, {3, 12, 36}, {1,5,15} + }; + for (int i = 0; i < 10; i++) { + int[] shardSplits = randomFrom(possibleValues); + assertEquals(shardSplits[0], (shardSplits[0] * shardSplits[1]) / shardSplits[1]); + assertEquals(shardSplits[1], (shardSplits[1] * shardSplits[2]) / shardSplits[2]); + IndexMetaData metaData = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[0]) + .numberOfReplicas(1).setRoutingNumShards(shardSplits[2]).build(); + String term = randomAlphaOfLength(10); + final int shard = OperationRouting.generateShardId(metaData, term, null); + IndexMetaData split = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[1]) + .numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + int shrunkShard = OperationRouting.generateShardId(split, term, null); + + ShardId shardId = IndexMetaData.selectSplitShard(shrunkShard, metaData, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + + split = IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(shardSplits[2]).numberOfReplicas(1) + .setRoutingNumShards(shardSplits[2]).build(); + shrunkShard = OperationRouting.generateShardId(split, term, null); + shardId = IndexMetaData.selectSplitShard(shrunkShard, metaData, split.getNumberOfShards()); + assertNotNull(shardId); + assertEquals(shard, shardId.getId()); + } + } + public void testPartitionedIndex() { // make sure the same routing value always has each _id fall within the configured partition size for (int shards = 1; shards < 5; shards++) { @@ -373,7 +403,7 @@ public void testPreferNodes() throws InterruptedException, IOException { terminate(threadPool); } } - + public void testFairSessionIdPreferences() throws InterruptedException, IOException { // Ensure that a user session is re-routed back to same nodes for // subsequent searches and that the nodes are selected fairly i.e. @@ -424,13 +454,13 @@ public void testFairSessionIdPreferences() throws InterruptedException, IOExcept assertThat("Search should use more than one of the nodes", selectedNodes.size(), greaterThan(1)); } } - + // Regression test for the routing logic - implements same hashing logic private ShardIterator duelGetShards(ClusterState clusterState, ShardId shardId, String sessionId) { final IndexShardRoutingTable indexShard = clusterState.getRoutingTable().shardRoutingTable(shardId.getIndexName(), shardId.getId()); int routingHash = Murmur3HashFunction.hash(sessionId); routingHash = 31 * routingHash + indexShard.shardId.hashCode(); - return indexShard.activeInitializingShardsIt(routingHash); + return indexShard.activeInitializingShardsIt(routingHash); } public void testThatOnlyNodesSupportNodeIds() throws InterruptedException, IOException { diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java new file mode 100644 index 0000000000000..507558608956c --- /dev/null +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java @@ -0,0 +1,239 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.Version; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ESAllocationTestCase; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.cluster.routing.allocation.decider.ResizeAllocationDecider; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.gateway.TestGatewayAllocator; + +import java.util.Arrays; +import java.util.Collections; + +import static org.elasticsearch.cluster.routing.ShardRoutingState.INITIALIZING; +import static org.elasticsearch.cluster.routing.ShardRoutingState.STARTED; +import static org.elasticsearch.cluster.routing.ShardRoutingState.UNASSIGNED; + + +public class ResizeAllocationDeciderTests extends ESAllocationTestCase { + + private AllocationService strategy; + + @Override + public void setUp() throws Exception { + super.setUp(); + strategy = new AllocationService(Settings.builder().build(), new AllocationDeciders(Settings.EMPTY, + Collections.singleton(new ResizeAllocationDecider(Settings.EMPTY))), + new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), EmptyClusterInfoService.INSTANCE); + } + + private ClusterState createInitialClusterState(boolean startShards) { + MetaData.Builder metaBuilder = MetaData.builder(); + metaBuilder.put(IndexMetaData.builder("source").settings(settings(Version.CURRENT)) + .numberOfShards(2).numberOfReplicas(0).setRoutingNumShards(16)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + routingTableBuilder.addAsNew(metaData.index("source")); + + RoutingTable routingTable = routingTableBuilder.build(); + ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) + .metaData(metaData).routingTable(routingTable).build(); + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))) + .build(); + RoutingTable prevRoutingTable = routingTable; + routingTable = strategy.reroute(clusterState, "reroute", false).routingTable(); + clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); + + assertEquals(prevRoutingTable.index("source").shards().size(), 2); + assertEquals(prevRoutingTable.index("source").shard(0).shards().get(0).state(), UNASSIGNED); + assertEquals(prevRoutingTable.index("source").shard(1).shards().get(0).state(), UNASSIGNED); + + + assertEquals(routingTable.index("source").shards().size(), 2); + + assertEquals(routingTable.index("source").shard(0).shards().get(0).state(), INITIALIZING); + assertEquals(routingTable.index("source").shard(1).shards().get(0).state(), INITIALIZING); + + + if (startShards) { + clusterState = strategy.applyStartedShards(clusterState, + Arrays.asList(routingTable.index("source").shard(0).shards().get(0), + routingTable.index("source").shard(1).shards().get(0))); + routingTable = clusterState.routingTable(); + assertEquals(routingTable.index("source").shards().size(), 2); + assertEquals(routingTable.index("source").shard(0).shards().get(0).state(), STARTED); + assertEquals(routingTable.index("source").shard(1).shards().get(0).state(), STARTED); + + } + return clusterState; + } + + public void testNonResizeRouting() { + ClusterState clusterState = createInitialClusterState(true); + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting("non-resize", 0, null, true, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + } + + public void testShrink() { // we don't handle shrink yet + ClusterState clusterState = createInitialClusterState(true); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(1).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, 0), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.ALWAYS, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } + + public void testSourceNotActive() { + ClusterState clusterState = createInitialClusterState(false); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + + routingAllocation.debugDecision(true); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node0"), + routingAllocation).getExplanation()); + assertEquals("source primary shard [[source][" + sourceShardId + "]] is not active", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + } + + public void testSourcePrimaryActive() { + ClusterState clusterState = createInitialClusterState(true); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + + String allowedNode = clusterState.getRoutingTable().index("source").shard(sourceShardId).primaryShard().currentNodeId(); + + if ("node1".equals(allowedNode)) { + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } else { + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + } + + routingAllocation.debugDecision(true); + assertEquals("source primary is active", resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + + if ("node1".equals(allowedNode)) { + assertEquals("source primary is allocated on this node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("source primary is allocated on another node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } else { + assertEquals("source primary is allocated on another node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("source primary is allocated on this node", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java new file mode 100644 index 0000000000000..7351372620fc9 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/shard/ShardSplittingQueryTests.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.shard; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.RoutingFieldMapper; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +public class ShardSplittingQueryTests extends ESTestCase { + + public void testSplitOnID() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), null); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + + + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + public void testSplitOnRouting() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, null, routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + public void testSplitOnIdOrRouting() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + if (randomBoolean()) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, null, routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } else { + int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), null); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + + public void testSplitOnRoutingPartitioned() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + RandomIndexWriter writer = new RandomIndexWriter(random(), dir); + int numShards = randomIntBetween(2, 10); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .routingPartitionSize(randomIntBetween(1, 10)) + .numberOfReplicas(0).build(); + int targetShardId = randomIntBetween(0, numShards-1); + for (int j = 0; j < numDocs; j++) { + String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 5); + final int shardId = OperationRouting.generateShardId(metaData, Integer.toString(j), routing); + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new StringField(RoutingFieldMapper.NAME, routing, Field.Store.YES), + new SortedNumericDocValuesField("shard_id", shardId) + )); + } + writer.commit(); + writer.close(); + assertSplit(dir, metaData, targetShardId); + dir.close(); + } + + + + + void assertSplit(Directory dir, IndexMetaData metaData, int targetShardId) throws IOException { + try (IndexReader reader = DirectoryReader.open(dir)) { + IndexSearcher searcher = new IndexSearcher(reader); + searcher.setQueryCache(null); + final boolean needsScores = false; + final Weight splitWeight = searcher.createNormalizedWeight(new ShardSplittingQuery(metaData, targetShardId), needsScores); + final List leaves = reader.leaves(); + for (final LeafReaderContext ctx : leaves) { + Scorer scorer = splitWeight.scorer(ctx); + DocIdSetIterator iterator = scorer.iterator(); + SortedNumericDocValues shard_id = ctx.reader().getSortedNumericDocValues("shard_id"); + int doc; + while ((doc = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + while (shard_id.nextDoc() < doc) { + long shardID = shard_id.nextValue(); + assertEquals(shardID, targetShardId); + } + assertEquals(shard_id.docID(), doc); + long shardID = shard_id.nextValue(); + BytesRef id = reader.document(doc).getBinaryValue("_id"); + String actualId = Uid.decodeId(id.bytes, id.offset, id.length); + assertNotEquals(ctx.reader() + " docID: " + doc + " actualID: " + actualId, shardID, targetShardId); + } + } + } + } +} diff --git a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java index 8d3ac8433d17d..05b092ff3a461 100644 --- a/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java +++ b/core/src/test/java/org/elasticsearch/index/shard/StoreRecoveryTests.java @@ -25,17 +25,28 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.routing.OperationRouting; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.seqno.SequenceNumbers; import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.test.ESTestCase; @@ -46,7 +57,9 @@ import java.nio.file.attribute.BasicFileAttributes; import java.security.AccessControlException; import java.util.Arrays; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import static org.hamcrest.CoreMatchers.equalTo; @@ -87,7 +100,7 @@ public void testAddIndices() throws IOException { Directory target = newFSDirectory(createTempDir()); final long maxSeqNo = randomNonNegativeLong(); final long maxUnsafeAutoIdTimestamp = randomNonNegativeLong(); - storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp); + storeRecovery.addIndices(indexStats, target, indexSort, dirs, maxSeqNo, maxUnsafeAutoIdTimestamp, null, 0, false); int numFiles = 0; Predicate filesFilter = (f) -> f.startsWith("segments") == false && f.equals("write.lock") == false && f.startsWith("extra") == false; @@ -122,6 +135,99 @@ public void testAddIndices() throws IOException { IOUtils.close(dirs); } + public void testSplitShard() throws IOException { + Directory dir = newFSDirectory(createTempDir()); + final int numDocs = randomIntBetween(50, 100); + final Sort indexSort; + if (randomBoolean()) { + indexSort = new Sort(new SortedNumericSortField("num", SortField.Type.LONG, true)); + } else { + indexSort = null; + } + int id = 0; + IndexWriterConfig iwc = newIndexWriterConfig() + .setMergePolicy(NoMergePolicy.INSTANCE) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + if (indexSort != null) { + iwc.setIndexSort(indexSort); + } + IndexWriter writer = new IndexWriter(dir, iwc); + for (int j = 0; j < numDocs; j++) { + writer.addDocument(Arrays.asList( + new StringField(IdFieldMapper.NAME, Uid.encodeId(Integer.toString(j)), Field.Store.YES), + new SortedNumericDocValuesField("num", randomLong()) + )); + } + + writer.commit(); + writer.close(); + StoreRecovery storeRecovery = new StoreRecovery(new ShardId("foo", "bar", 1), logger); + RecoveryState.Index indexStats = new RecoveryState.Index(); + Directory target = newFSDirectory(createTempDir()); + final long maxSeqNo = randomNonNegativeLong(); + final long maxUnsafeAutoIdTimestamp = randomNonNegativeLong(); + int numShards = randomIntBetween(2, 10); + int targetShardId = randomIntBetween(0, numShards-1); + IndexMetaData metaData = IndexMetaData.builder("test") + .settings(Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(numShards) + .setRoutingNumShards(numShards * 1000000) + .numberOfReplicas(0).build(); + storeRecovery.addIndices(indexStats, target, indexSort, new Directory[] {dir}, maxSeqNo, maxUnsafeAutoIdTimestamp, metaData, + targetShardId, true); + + + SegmentInfos segmentCommitInfos = SegmentInfos.readLatestCommit(target); + final Map userData = segmentCommitInfos.getUserData(); + assertThat(userData.get(SequenceNumbers.MAX_SEQ_NO), equalTo(Long.toString(maxSeqNo))); + assertThat(userData.get(SequenceNumbers.LOCAL_CHECKPOINT_KEY), equalTo(Long.toString(maxSeqNo))); + assertThat(userData.get(InternalEngine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID), equalTo(Long.toString(maxUnsafeAutoIdTimestamp))); + for (SegmentCommitInfo info : segmentCommitInfos) { // check that we didn't merge + assertEquals("all sources must be flush", + info.info.getDiagnostics().get("source"), "flush"); + if (indexSort != null) { + assertEquals(indexSort, info.info.getIndexSort()); + } + } + + iwc = newIndexWriterConfig() + .setMergePolicy(NoMergePolicy.INSTANCE) + .setOpenMode(IndexWriterConfig.OpenMode.CREATE); + if (indexSort != null) { + iwc.setIndexSort(indexSort); + } + writer = new IndexWriter(target, iwc); + writer.forceMerge(1, true); + writer.commit(); + writer.close(); + + DirectoryReader reader = DirectoryReader.open(target); + for (LeafReaderContext ctx : reader.leaves()) { + LeafReader leafReader = ctx.reader(); + Terms terms = leafReader.terms(IdFieldMapper.NAME); + TermsEnum iterator = terms.iterator(); + BytesRef ref; + while((ref = iterator.next()) != null) { + String value = ref.utf8ToString(); + assertEquals("value has wrong shards: " + value, targetShardId, OperationRouting.generateShardId(metaData, value, null)); + } + for (int i = 0; i < numDocs; i++) { + ref = new BytesRef(Integer.toString(i)); + int shardId = OperationRouting.generateShardId(metaData, ref.utf8ToString(), null); + if (shardId == targetShardId) { + assertTrue(ref.utf8ToString() + " is missing", terms.iterator().seekExact(ref)); + } else { + assertFalse(ref.utf8ToString() + " was found but shouldn't", terms.iterator().seekExact(ref)); + } + + } + } + + reader.close(); + target.close(); + IOUtils.close(dir); + } + public void testStatsDirWrapper() throws IOException { Directory dir = newDirectory(); Directory target = newDirectory(); diff --git a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java index b23ce6a9286bb..07a73a09f4ab4 100644 --- a/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java +++ b/core/src/test/java/org/elasticsearch/routing/PartitionedRoutingIT.java @@ -105,7 +105,7 @@ public void testShrinking() throws Exception { index = "index_" + currentShards; logger.info("--> shrinking index [" + previousIndex + "] to [" + index + "]"); - client().admin().indices().prepareShrinkIndex(previousIndex, index) + client().admin().indices().prepareResizeIndex(previousIndex, index) .setSettings(Settings.builder() .put("index.number_of_shards", currentShards) .put("index.number_of_replicas", numberOfReplicas()) diff --git a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java index 17412f8f724e4..5341b268544e7 100644 --- a/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java +++ b/core/src/test/java/org/elasticsearch/snapshots/DedicatedClusterSnapshotRestoreIT.java @@ -950,7 +950,7 @@ public void testRestoreShrinkIndex() throws Exception { logger.info("--> shrink the index"); assertAcked(client.admin().indices().prepareUpdateSettings(sourceIdx) .setSettings(Settings.builder().put("index.blocks.write", true)).get()); - assertAcked(client.admin().indices().prepareShrinkIndex(sourceIdx, shrunkIdx).get()); + assertAcked(client.admin().indices().prepareResizeIndex(sourceIdx, shrunkIdx).get()); logger.info("--> snapshot the shrunk index"); CreateSnapshotResponse createResponse = client.admin().cluster() diff --git a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java index 6dd4fa384e99b..e5081481859ab 100644 --- a/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java +++ b/core/src/test/java/org/elasticsearch/test/search/aggregations/bucket/SharedSignificantTermsTestMethods.java @@ -50,7 +50,7 @@ public class SharedSignificantTermsTestMethods { public static void aggregateAndCheckFromSeveralShards(ESIntegTestCase testCase) throws ExecutionException, InterruptedException { String type = ESTestCase.randomBoolean() ? "text" : "keyword"; - String settings = "{\"index.number_of_shards\": 5, \"index.number_of_replicas\": 0}"; + String settings = "{\"index.number_of_shards\": 7, \"index.number_of_replicas\": 0}"; index01Docs(type, settings, testCase); testCase.ensureGreen(); testCase.logClusterState(); diff --git a/docs/reference/indices.asciidoc b/docs/reference/indices.asciidoc index 873021c420636..70d3b19d3fd59 100644 --- a/docs/reference/indices.asciidoc +++ b/docs/reference/indices.asciidoc @@ -16,6 +16,7 @@ index settings, aliases, mappings, and index templates. * <> * <> * <> +* <> * <> [float] diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc new file mode 100644 index 0000000000000..a83f6030103f4 --- /dev/null +++ b/docs/reference/indices/split-index.asciidoc @@ -0,0 +1,158 @@ +[[indices-split-index]] +== Split Index + +The split index API allows you to split an existing index into a new index +with multiple of it's primary shards. Similarly to the <> +where the number of primary shards in the shrunk index must be a factor of the source index. +The `_split` API requires the source index to be created with a routing shard factor in order +to be split. (Note: this requirement might be remove in future releases) For example an index +with `8` primary shards and a `index.routing_shards_factor` of `2` can be split into `16` +primary shards or an index with `1` primary shard and `index.routing_shards_factor` of `64` +can be split into `2`, `4`, `8`, `16`, `32` or `64`. +Before splitting, a (primary) copy of every shard in the index must be active in the cluster. + +Splitting works as follows: + +* First, it creates a new target index with the same definition as the source + index, but with a larger number of primary shards. + +* Then it hard-links segments from the source index into the target index. (If + the file system doesn't support hard-linking, then all segments are copied + into the new index, which is a much more time consuming process.) + +* Once the low level files are created all documents will be `hashed` again to delete + documents that belong in a different shard. + +* Finally, it recovers the target index as though it were a closed index which + had just been re-opened. + +[float] +=== Preparing an index for splitting + +Create an index with a routing shards factor: + +[source,js] +-------------------------------------------------- +PUT my_source_index +{ + "settings": { + "index.number_of_shards" : 1, + "index.routing_shards_factor" : 2 <1> + } +} +------------------------------------------------- +// CONSOLE + +<1> Allows to split the index into twice as many shards ie. it allows + for a single split operation. The split will allow to go from the + default of 5 shards to 10 shards. + +In order to split an index, the index must be marked as read-only, +and have <> `green`. + +This can be achieved with the following request: + +[source,js] +-------------------------------------------------- +PUT /my_source_index/_settings +{ + "settings": { + "index.blocks.write": true <1> + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +<1> Prevents write operations to this index while still allowing metadata + changes like deleting the index. + +[float] +=== Spitting an index + +To split `my_source_index` into a new index called `my_target_index`, issue +the following request: + +[source,js] +-------------------------------------------------- +POST my_source_index/_split/my_target_index +{ + "settings": { + "index.number_of_shards": 2 + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The above request returns immediately once the target index has been added to +the cluster state -- it doesn't wait for the split operation to start. + +[IMPORTANT] +===================================== + +Indices can only be split if they satisfy the following requirements: + +* the target index must not exist + +* The index must have less primary shards than the target index. + +* The number of primary shards in the target index must be a power of two + factor of the number of primary shards in the source index. + +* The node handling the split process must have sufficient free disk space to + accommodate a second copy of the existing index. + +===================================== + +The `_split` API is similar to the <> +and accepts `settings` and `aliases` parameters for the target index: + +[source,js] +-------------------------------------------------- +POST my_source_index/_split/my_target_index +{ + "settings": { + "index.number_of_shards": 5 <1> + }, + "aliases": { + "my_search_indices": {} + } +} +-------------------------------------------------- +// CONSOLE +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.routing_shards_factor" : 5}}\n/] + +<1> The number of shards in the target index. This must be a factor of the + number of shards in the source index. + + +NOTE: Mappings may not be specified in the `_split` request, and all +`index.analysis.*` and `index.similarity.*` settings will be overwritten with +the settings from the source index. + +[float] +=== Monitoring the split process + +The split process can be monitored with the <>, or the <> can be used to wait +until all primary shards have been allocated by setting the `wait_for_status` +parameter to `yellow`. + +The `_split` API returns as soon as the target index has been added to the +cluster state, before any shards have been allocated. At this point, all +shards are in the state `unassigned`. If, for any reason, the target index +can't be allocated, its primary shard will remain `unassigned` until it +can be allocated on that node. + +Once the primary shard is allocated, it moves to state `initializing`, and the +split process begins. When the split operation completes, the shard will +become `active`. At that point, Elasticsearch will try to allocate any +replicas and may decide to relocate the primary shard to another node. + +[float] +=== Wait For Active Shards + +Because the split operation creates a new index to split the shards to, +the <> setting +on index creation applies to the split index action as well. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json new file mode 100644 index 0000000000000..a79fa7b708269 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json @@ -0,0 +1,39 @@ +{ + "indices.split": { + "documentation": "http://www.elastic.co/guide/en/elasticsearch/reference/master/indices-split-index.html", + "methods": ["PUT", "POST"], + "url": { + "path": "/{index}/_split/{target}", + "paths": ["/{index}/_split/{target}"], + "parts": { + "index": { + "type" : "string", + "required" : true, + "description" : "The name of the source index to split" + }, + "target": { + "type" : "string", + "required" : true, + "description" : "The name of the target index to split into" + } + }, + "params": { + "timeout": { + "type" : "time", + "description" : "Explicit operation timeout" + }, + "master_timeout": { + "type" : "time", + "description" : "Specify timeout for connection to master" + }, + "wait_for_active_shards": { + "type" : "string", + "description" : "Set the number of active shards to wait for on the shrunken index before the operation returns." + } + } + }, + "body": { + "description" : "The configuration for the target index (`settings` and `aliases`)" + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml new file mode 100644 index 0000000000000..6ce39e311853a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -0,0 +1,101 @@ +--- +"Split index via API": + - skip: + version: " - 6.99.99" + reason: Added in 7.0.0 + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + index.number_of_shards: 1 + index.number_of_replicas: 0 + index.routing_shards_factor: 2 + - do: + index: + index: source + type: doc + id: "1" + body: { "foo": "hello world" } + + - do: + index: + index: source + type: doc + id: "2" + body: { "foo": "hello world 2" } + + - do: + index: + index: source + type: doc + id: "3" + body: { "foo": "hello world 3" } + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do the actual split + - do: + indices.split: + index: "source" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_replicas: 0 + index.number_of_shards: 2 + + - do: + cluster.health: + wait_for_status: green + + - do: + get: + index: target + type: doc + id: "1" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "1" } + - match: { _source: { foo: "hello world" } } + + + - do: + get: + index: target + type: doc + id: "2" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "2" } + - match: { _source: { foo: "hello world 2" } } + + + - do: + get: + index: target + type: doc + id: "3" + + - match: { _index: target } + - match: { _type: doc } + - match: { _id: "3" } + - match: { _source: { foo: "hello world 3" } } + + + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml new file mode 100644 index 0000000000000..790f7d6d18402 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -0,0 +1,72 @@ +--- +"Split index ignores target template mapping": + - skip: + version: " - 6.99.99" + reason: added in 7.0.0 + + # create index + - do: + indices.create: + index: source + wait_for_active_shards: 1 + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + index.routing_shards_factor: 2 + mappings: + test: + properties: + count: + type: text + + # index document + - do: + index: + index: source + type: test + id: "1" + body: { "count": "1" } + + # create template matching shrink target + - do: + indices.put_template: + name: tpl1 + body: + index_patterns: targ* + mappings: + test: + properties: + count: + type: integer + + # make it read-only + - do: + indices.put_settings: + index: source + body: + index.blocks.write: true + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + index: source + + # now we do the actual split + - do: + indices.split: + index: "source" + target: "target" + wait_for_active_shards: 1 + master_timeout: 10s + body: + settings: + index.number_of_shards: 2 + index.number_of_replicas: 0 + + - do: + cluster.health: + wait_for_status: green + + diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java index a4ac6fad241a5..2291c3d39e200 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/routing/TestShardRouting.java @@ -39,6 +39,10 @@ public static ShardRouting newShardRouting(String index, int shardId, String cur return newShardRouting(new ShardId(index, IndexMetaData.INDEX_UUID_NA_VALUE, shardId), currentNodeId, primary, state); } + public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, RecoverySource recoverySource, ShardRoutingState state) { + return new ShardRouting(shardId, currentNodeId, null, primary, state, recoverySource, buildUnassignedInfo(state), buildAllocationId(state), -1); + } + public static ShardRouting newShardRouting(ShardId shardId, String currentNodeId, boolean primary, ShardRoutingState state) { return new ShardRouting(shardId, currentNodeId, null, primary, state, buildRecoveryTarget(primary, state), buildUnassignedInfo(state), buildAllocationId(state), -1); } From 658d31aeeada7c31636665eb213ee8b585adcc1d Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 9 Oct 2017 19:14:31 +0200 Subject: [PATCH 02/13] fix docs test --- docs/reference/indices/split-index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index a83f6030103f4..bf1434f525d80 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -121,7 +121,7 @@ POST my_source_index/_split/my_target_index } -------------------------------------------------- // CONSOLE -// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.routing_shards_factor" : 5}}\n/] +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.routing_shards_factor" : 5, "index.number_of_shards": "1"}}\n/] <1> The number of shards in the target index. This must be a factor of the number of shards in the source index. From 68d2fa6714a87b19104c552e947a1a61f0bba732 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 12 Oct 2017 15:52:26 +0200 Subject: [PATCH 03/13] apply feedback from @jpoutz --- .../org/elasticsearch/index/mapper/Uid.java | 56 ++++++++--------- .../index/shard/ShardSplittingQuery.java | 62 ++++++++----------- 2 files changed, 52 insertions(+), 66 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java index dae320511e5dd..1d5293259317f 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/Uid.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/Uid.java @@ -22,10 +22,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.UnicodeUtil; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.lucene.BytesRefs; -import java.io.ByteArrayInputStream; import java.util.Arrays; import java.util.Base64; import java.util.Collection; @@ -137,36 +135,36 @@ static boolean isURLBase64WithoutPadding(String id) { // 'xxx=' and 'xxx' could be considered the same id final int length = id.length(); switch (length & 0x03) { - case 0: - break; - case 1: - return false; - case 2: - // the last 2 symbols (12 bits) are encoding 1 byte (8 bits) - // so the last symbol only actually uses 8-6=2 bits and can only take 4 values - char last = id.charAt(length - 1); - if (last != 'A' && last != 'Q' && last != 'g' && last != 'w') { + case 0: + break; + case 1: return false; - } - break; - case 3: - // The last 3 symbols (18 bits) are encoding 2 bytes (16 bits) - // so the last symbol only actually uses 16-12=4 bits and can only take 16 values - last = id.charAt(length - 1); - if (last != 'A' && last != 'E' && last != 'I' && last != 'M' && last != 'Q'&& last != 'U'&& last != 'Y' + case 2: + // the last 2 symbols (12 bits) are encoding 1 byte (8 bits) + // so the last symbol only actually uses 8-6=2 bits and can only take 4 values + char last = id.charAt(length - 1); + if (last != 'A' && last != 'Q' && last != 'g' && last != 'w') { + return false; + } + break; + case 3: + // The last 3 symbols (18 bits) are encoding 2 bytes (16 bits) + // so the last symbol only actually uses 16-12=4 bits and can only take 16 values + last = id.charAt(length - 1); + if (last != 'A' && last != 'E' && last != 'I' && last != 'M' && last != 'Q'&& last != 'U'&& last != 'Y' && last != 'c'&& last != 'g'&& last != 'k' && last != 'o' && last != 's' && last != 'w' && last != '0' && last != '4' && last != '8') { - return false; - } - break; - default: - // number & 0x03 is always in [0,3] - throw new AssertionError("Impossible case"); + return false; + } + break; + default: + // number & 0x03 is always in [0,3] + throw new AssertionError("Impossible case"); } for (int i = 0; i < length; ++i) { final char c = id.charAt(i); final boolean allowed = - (c >= '0' && c <= '9') || + (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '_'; @@ -272,9 +270,9 @@ private static String decodeUtf8Id(byte[] idBytes, int offset, int length) { private static String decodeBase64Id(byte[] idBytes, int offset, int length) { assert Byte.toUnsignedInt(idBytes[offset]) <= BASE64_ESCAPE; if (Byte.toUnsignedInt(idBytes[offset]) == BASE64_ESCAPE) { - idBytes = Arrays.copyOfRange(idBytes, offset + 1, length); - } else if (idBytes.length != length || offset != 0) { - idBytes = Arrays.copyOfRange(idBytes, offset, length); + idBytes = Arrays.copyOfRange(idBytes, offset + 1, offset + length); + } else if ((idBytes.length == length && offset == 0) == false) { // no need to copy if it's not a slice + idBytes = Arrays.copyOfRange(idBytes, offset, offset + length); } return Base64.getUrlEncoder().withoutPadding().encodeToString(idBytes); } @@ -282,7 +280,7 @@ private static String decodeBase64Id(byte[] idBytes, int offset, int length) { /** Decode an indexed id back to its original form. * @see #encodeId */ public static String decodeId(byte[] idBytes) { - return decodeId(idBytes, 0, idBytes.length); + return decodeId(idBytes, 0, idBytes.length); } /** Decode an indexed id back to its original form. diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index 15f050a80173b..f1afc004bd7d1 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -31,11 +31,13 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.apache.lucene.util.BitSetIterator; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.OperationRouting; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -56,6 +58,10 @@ final class ShardSplittingQuery extends Query { private final int shardId; ShardSplittingQuery(IndexMetaData indexMetaData, int shardId) { + if (indexMetaData.getCreationVersion().before(Version.V_6_0_0_rc2)) { + throw new IllegalArgumentException("Splitting query can only be executed on an index created with version " + + Version.V_6_0_0_rc2 + " or higher"); + } this.indexMetaData = indexMetaData; this.shardId = shardId; } @@ -87,7 +93,7 @@ public Scorer scorer(LeafReaderContext context) throws IOException { Bits liveDocs = leafReader.getLiveDocs(); Visitor visitor = new Visitor(); return new ConstantScoreScorer(this, score(), - new RoutingPartitionedDocIdSetIterator(leafReader, liveDocs, visitor)); + new RoutingPartitionedDocIdSetIterator(leafReader, visitor)); } else { // in the _routing case we first go and find all docs that have a routing value and mark the ones we have to delete findSplitDocs(RoutingFieldMapper.NAME, ref -> { @@ -111,6 +117,8 @@ public Scorer scorer(LeafReaderContext context) throws IOException { } return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length())); } + + }; } @@ -121,12 +129,12 @@ public String toString(String field) { @Override public boolean equals(Object o) { - return sameClassAs(o); + throw new UnsupportedOperationException("only use this query for deleting documents"); } @Override public int hashCode() { - return classHash(); + throw new UnsupportedOperationException("only use this query for deleting documents"); } private static void findSplitDocs(String idField, Predicate includeInShard, @@ -184,6 +192,7 @@ public void stringField(FieldInfo fieldInfo, byte[] value) throws IOException { @Override public Status needsField(FieldInfo fieldInfo) throws IOException { + // we don't support 5.x so no need for the uid field switch (fieldInfo.name) { case IdFieldMapper.NAME: case RoutingFieldMapper.NAME: @@ -191,57 +200,36 @@ public Status needsField(FieldInfo fieldInfo) throws IOException { return Status.YES; default: return leftToVisit == 0 ? Status.STOP : Status.NO; - } } } /** - * This DISI visits every live doc and selects all docs that don't belong into this - * shard based on their id and rounting value. This is only used in a routing partitioned index. + * This two phase iterator visits every live doc and selects all docs that don't belong into this + * shard based on their id and routing value. This is only used in a routing partitioned index. */ - private final class RoutingPartitionedDocIdSetIterator extends DocIdSetIterator { + private final class RoutingPartitionedDocIdSetIterator extends TwoPhaseIterator { private final LeafReader leafReader; - private final Bits liveDocs; private final Visitor visitor; - private int doc; - RoutingPartitionedDocIdSetIterator(LeafReader leafReader, Bits liveDocs, Visitor visitor) { + RoutingPartitionedDocIdSetIterator(LeafReader leafReader, Visitor visitor) { + super(DocIdSetIterator.all(leafReader.maxDoc())); // we iterate all live-docs this.leafReader = leafReader; - this.liveDocs = liveDocs; this.visitor = visitor; - doc = -1; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() throws IOException { - while (++doc < leafReader.maxDoc()) { - if (liveDocs == null || liveDocs.get(doc)) { - visitor.reset(); - leafReader.document(doc, visitor); - int targetShardId = OperationRouting.generateShardId(indexMetaData, visitor.id, visitor.routing); - if (targetShardId != shardId) { // move to next doc if we can keep it - return doc; - } - } - } - return doc = DocIdSetIterator.NO_MORE_DOCS; } @Override - public int advance(int target) throws IOException { - while (nextDoc() < target) {} - return doc; + public boolean matches() throws IOException { + int doc = approximation.docID(); + visitor.reset(); + leafReader.document(doc, visitor); + int targetShardId = OperationRouting.generateShardId(indexMetaData, visitor.id, visitor.routing); + return targetShardId != shardId; } @Override - public long cost() { - return leafReader.maxDoc(); + public float matchCost() { + return 42; // that's obvious, right? } } } From e41efd80f9bb2ba268ec66aebfd6c92f886b3f68 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 12 Oct 2017 16:26:16 +0200 Subject: [PATCH 04/13] add a working hashcode / equals method to ShardSplittingQuery --- .../index/shard/ShardSplittingQuery.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index f1afc004bd7d1..04924f5113922 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -129,12 +129,20 @@ public String toString(String field) { @Override public boolean equals(Object o) { - throw new UnsupportedOperationException("only use this query for deleting documents"); + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + ShardSplittingQuery that = (ShardSplittingQuery) o; + + if (shardId != that.shardId) return false; + return indexMetaData.equals(that.indexMetaData); } @Override public int hashCode() { - throw new UnsupportedOperationException("only use this query for deleting documents"); + int result = indexMetaData.hashCode(); + result = 31 * result + shardId; + return result; } private static void findSplitDocs(String idField, Predicate includeInShard, From 7f33942c944feb16a5f923fa665929db3a8c35aa Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 12 Oct 2017 17:01:35 +0200 Subject: [PATCH 05/13] fix test --- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 34c45a8cd2414..d68e392849443 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -219,7 +219,7 @@ public void testValidateSplitIndex() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); int targetShards; do { - targetShards = randomIntBetween(numShards, 100); + targetShards = randomIntBetween(numShards+1, 100); } while (isSplitable(numShards, targetShards) == false); MetaDataCreateIndexService.validateSplitIndex(clusterState, "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", targetShards).build()); From d25ee873c13465ebabedb585992ea63539334939 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 12 Oct 2017 21:13:48 +0200 Subject: [PATCH 06/13] apply review commetns from @ywelsch --- .../indices/shrink/TransportResizeAction.java | 2 +- .../cluster/metadata/IndexMetaData.java | 30 ++++++------ .../metadata/MetaDataCreateIndexService.java | 5 +- .../decider/ResizeAllocationDecider.java | 46 ++++++++++--------- .../elasticsearch/index/shard/IndexShard.java | 14 ++++-- .../admin/indices/create/SplitIndexIT.java | 2 +- .../MetaDataCreateIndexServiceTests.java | 4 +- 7 files changed, 58 insertions(+), 45 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index a3b7c5b84f1bf..71751599956dd 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -165,7 +165,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi return new CreateIndexClusterStateUpdateRequest(targetIndex, cause, targetIndex.index(), targetIndexName, true) // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be - // applied once we took the snapshot and if somebody fucks things up and switches the index read/write and adds docs we miss + // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we miss // the mappings for everything is corrupted and hard to debug .ackTimeout(targetIndex.timeout()) .masterNodeTimeout(targetIndex.masterNodeTimeout()) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index ac80415fe5862..a1e6bba27516d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1323,7 +1323,8 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet } int numSourceShards = sourceIndexMetadata.getNumberOfShards(); if (numSourceShards > numTargetShards) { - throw new IllegalArgumentException("the number of source shards must be less that the number of target shards"); + throw new IllegalArgumentException("the number of source shards [" + numSourceShards + + "] must be less that the number of target shards [" + numTargetShards + "]"); } int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); @@ -1331,7 +1332,7 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet } /** - * Selects the source shards fro a local shard recovery. This might either be a split or a shrink operation. + * Selects the source shards for a local shard recovery. This might either be a split or a shrink operation. * @param shardId the target shard ID to select the source shards for * @param sourceIndexMetadata the source metadata * @param numTargetShards the number of target shards @@ -1381,22 +1382,23 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI * are not divisible by the number of target shards. */ public static int getRoutingFactor(int sourceNumberOfShards, int targetNumberOfShards) { - if (sourceNumberOfShards < targetNumberOfShards) { - int spare = sourceNumberOfShards; - sourceNumberOfShards = targetNumberOfShards; - targetNumberOfShards = spare; - } - - int factor = sourceNumberOfShards / targetNumberOfShards; - if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { - if (sourceNumberOfShards < targetNumberOfShards) { + final int factor; + if (sourceNumberOfShards < targetNumberOfShards) { // split + factor = targetNumberOfShards / sourceNumberOfShards; + if (factor * sourceNumberOfShards != targetNumberOfShards || factor <= 1) { throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + "factor of [" + targetNumberOfShards + "]"); } - throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + - "multiple of [" - + targetNumberOfShards + "]"); + } else if (sourceNumberOfShards > targetNumberOfShards) { // shrink + factor = sourceNumberOfShards / targetNumberOfShards; + if (factor * targetNumberOfShards != sourceNumberOfShards || factor <= 1) { + throw new IllegalArgumentException("the number of source shards [" + sourceNumberOfShards + "] must be a must be a " + + "multiple of [" + + targetNumberOfShards + "]"); + } + } else { + factor = 1; } return factor; } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index 4f26fac16a10b..d7be87b2f872c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -386,7 +386,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { Settings idxSettings = indexSettingsBuilder.build(); int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); int routingShardsFactor = IndexMetaData.INDEX_ROUTING_SHARDS_FACTOR_SETTING.get(idxSettings); - // we multiply the routing shares in order to split the shard going forward. + // we multiply the routing shards in order to split the shard going forward. // implementation wise splitting shards is really just an inverted shrink operation. routingNumShards = numShards * routingShardsFactor; } else { @@ -652,7 +652,8 @@ static void validateSplitIndex(ClusterState state, String sourceIndex, IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); } if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { - // ensure we have a single type. + // ensure we have a single type since this would make the splitting code considerably more complex + // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity throw new IllegalStateException("source index created version is too old to apply a split operation"); } diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java index 94855ac8261d2..40ef62cb0f423 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java @@ -19,6 +19,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; @@ -59,32 +60,35 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo(); if (unassignedInfo != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) { - // we only make decisions here if we have no unassigned info and we have to recover from another index ie. split / shrink + // we only make decisions here if we have an unassigned info and we have to recover from another index ie. split / shrink final IndexMetaData indexMetaData = allocation.metaData().getIndexSafe(shardRouting.index()); Index resizeSourceIndex = indexMetaData.getResizeSourceIndex(); assert resizeSourceIndex != null; - try { - IndexMetaData sourceIndexMetaData = allocation.metaData().getIndexSafe(resizeSourceIndex); - if (indexMetaData.getNumberOfShards() < sourceIndexMetaData.getNumberOfShards()) { - // this only handles splits so far. - return Decision.ALWAYS; - } - ShardId shardId = IndexMetaData.selectSplitShard(shardRouting.id(), sourceIndexMetaData, indexMetaData.getNumberOfShards()); - ShardRouting sourceShardRouting = allocation.routingTable().shardRoutingTable(shardId).primaryShard(); - if (sourceShardRouting.active() == false) { - return allocation.decision(Decision.NO, NAME, "source primary shard [%s] is not active", sourceShardRouting.shardId()); + if (allocation.metaData().index(resizeSourceIndex) == null) { + return allocation.decision(Decision.NO, NAME, "resize source index [%s] doesn't exists", resizeSourceIndex.toString()); + } + IndexMetaData sourceIndexMetaData = allocation.metaData().getIndexSafe(resizeSourceIndex); + if (indexMetaData.getNumberOfShards() < sourceIndexMetaData.getNumberOfShards()) { + // this only handles splits so far. + return Decision.ALWAYS; + } + + ShardId shardId = IndexMetaData.selectSplitShard(shardRouting.id(), sourceIndexMetaData, indexMetaData.getNumberOfShards()); + ShardRouting sourceShardRouting = allocation.routingNodes().activePrimary(shardId); + if (sourceShardRouting == null) { + return allocation.decision(Decision.NO, NAME, "source primary shard [%s] is not active", shardId); + } + if (node != null) { // we might get called from the 2 param canAllocate method.. + if (node.node().getVersion().before(Version.V_7_0_0_alpha1)) { + return allocation.decision(Decision.NO, NAME, "node [%s] is too old to split a shard", node.nodeId()); } - if (node != null) { // we might get called from the 2 param canAllocate method.. - if (sourceShardRouting.currentNodeId().equals(node.nodeId())) { - return allocation.decision(Decision.YES, NAME, "source primary is allocated on this node"); - } else { - return allocation.decision(Decision.NO, NAME, "source primary is allocated on another node"); - } + if (sourceShardRouting.currentNodeId().equals(node.nodeId())) { + return allocation.decision(Decision.YES, NAME, "source primary is allocated on this node"); } else { - return allocation.decision(Decision.YES, NAME, "source primary is active"); + return allocation.decision(Decision.NO, NAME, "source primary is allocated on another node"); } - } catch (IndexNotFoundException ex) { - return allocation.decision(Decision.NO, NAME, "resize source index [%s] doesn't exists", resizeSourceIndex.toString()); + } else { + return allocation.decision(Decision.YES, NAME, "source primary is active"); } } return super.canAllocate(shardRouting, node, allocation); @@ -93,8 +97,6 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing @Override public Decision canForceAllocatePrimary(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) { assert shardRouting.primary() : "must not call canForceAllocatePrimary on a non-primary shard " + shardRouting; - // check if we have passed the maximum retry threshold through canAllocate, - // if so, we don't want to force the primary allocation here return canAllocate(shardRouting, node, allocation); } } diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 4cd10caf9654e..49f7112c44e0a 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -139,6 +139,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.EnumSet; import java.util.List; import java.util.Locale; @@ -2040,17 +2041,24 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex(); final List startedShards = new ArrayList<>(); final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex); - final Set requiredShards = IndexMetaData.selectRecoverFromShards(shardId().id(), - sourceIndexService.getMetaData(), indexMetaData.getNumberOfShards()); - final int numShards = sourceIndexService != null ? requiredShards.size() : -1; + final Set requiredShards; + final int numShards; if (sourceIndexService != null) { + requiredShards = IndexMetaData.selectRecoverFromShards(shardId().id(), + sourceIndexService.getMetaData(), indexMetaData.getNumberOfShards()); for (IndexShard shard : sourceIndexService) { if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) { startedShards.add(shard); } } + numShards = requiredShards.size(); + } else { + numShards = -1; + requiredShards = Collections.emptySet(); } + if (numShards == startedShards.size()) { + assert requiredShards.isEmpty() == false; markAsRecovering("from local shards", recoveryState); // mark the shard as recovering on the cluster state thread threadPool.generic().execute(() -> { try { diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 76d5140a9bfdb..e627bec7b2a96 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -277,7 +277,7 @@ private static IndexMetaData indexMetaData(final Client client, final String ind public void testCreateSplitIndex() { internalCluster().ensureAtLeastNumDataNodes(2); - Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_alpha1, Version.CURRENT); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_6_0_0_rc2, Version.CURRENT); prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) .put("number_of_shards", 1) .put("index.version.created", version) diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index d68e392849443..72f4bd33ad56e 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -174,7 +174,7 @@ public void testValidateSplitIndex() { MetaDataCreateIndexService.validateSplitIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY) ).getMessage()); - assertEquals("the number of source shards must be less that the number of target shards", + assertEquals("the number of source shards [10] must be less that the number of target shards [5]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", 10, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 5).build()) @@ -189,7 +189,7 @@ public void testValidateSplitIndex() { ).getMessage()); - assertEquals("the number of source shards [4] must be a must be a multiple of [3]", + assertEquals("the number of source shards [3] must be a must be a factor of [4]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateSplitIndex(createClusterState("source", 3, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", From e825b08cc4ec8e10f8f8b0d95e0e66b5efe09bb4 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 12 Oct 2017 21:19:36 +0200 Subject: [PATCH 07/13] add missign comments --- .../java/org/elasticsearch/cluster/metadata/IndexMetaData.java | 3 ++- .../org/elasticsearch/cluster/routing/OperationRouting.java | 2 +- .../cluster/metadata/MetaDataCreateIndexServiceTests.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index a1e6bba27516d..b4ed372322989 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1359,7 +1359,8 @@ public static Set selectShrinkShards(int shardId, IndexMetaData sourceI + shardId); } if (sourceIndexMetadata.getNumberOfShards() < numTargetShards) { - throw new IllegalArgumentException("the number of target shards must be less that the number of source shards"); + throw new IllegalArgumentException("the number of target shards [" + numTargetShards + +"] must be less that the number of source shards [" + sourceIndexMetadata.getNumberOfShards() + "]"); } int routingFactor = getRoutingFactor(sourceIndexMetadata.getNumberOfShards(), numTargetShards); Set shards = new HashSet<>(routingFactor); diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java index fa30123a1b5cc..005600ceb4431 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/OperationRouting.java @@ -285,7 +285,7 @@ private static int calculateScaledShardId(IndexMetaData indexMetaData, String ef // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size // of original index to hash documents - return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor(); + return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor(); } } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 72f4bd33ad56e..45525b797ef68 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -104,7 +104,7 @@ public void testValidateShrinkIndex() { "target", Settings.EMPTY) ).getMessage()); - assertEquals("the number of target shards must be less that the number of source shards", + assertEquals("the number of target shards [10] must be less that the number of source shards [5]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 10).build()) From 3af28d1fb14ddf74fdfffca521f8c985c4cc8474 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Fri, 13 Oct 2017 09:44:34 +0200 Subject: [PATCH 08/13] add test to prevent allocation on old node --- .../ResizeAllocationDeciderTests.java | 54 +++++++++++++++++-- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java index 507558608956c..cb9919216bdd0 100644 --- a/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/routing/allocation/ResizeAllocationDeciderTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.gateway.TestGatewayAllocator; import java.util.Arrays; @@ -61,6 +62,10 @@ public void setUp() throws Exception { } private ClusterState createInitialClusterState(boolean startShards) { + return createInitialClusterState(startShards, Version.CURRENT); + } + + private ClusterState createInitialClusterState(boolean startShards, Version nodeVersion) { MetaData.Builder metaBuilder = MetaData.builder(); metaBuilder.put(IndexMetaData.builder("source").settings(settings(Version.CURRENT)) .numberOfShards(2).numberOfReplicas(0).setRoutingNumShards(16)); @@ -71,7 +76,8 @@ private ClusterState createInitialClusterState(boolean startShards) { RoutingTable routingTable = routingTableBuilder.build(); ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)) .metaData(metaData).routingTable(routingTable).build(); - clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))) + clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1", nodeVersion)).add(newNode + ("node2", nodeVersion))) .build(); RoutingTable prevRoutingTable = routingTable; routingTable = strategy.reroute(clusterState, "reroute", false).routingTable(); @@ -156,7 +162,7 @@ public void testSourceNotActive() { ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); - RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); int shardId = randomIntBetween(0, 3); int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource @@ -196,7 +202,7 @@ public void testSourcePrimaryActive() { ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); - RoutingAllocation routingAllocation = new RoutingAllocation(null, null, clusterState, null, 0); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); int shardId = randomIntBetween(0, 3); int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource @@ -236,4 +242,46 @@ public void testSourcePrimaryActive() { routingAllocation).getExplanation()); } } + + public void testAllocateOnOldNode() { + Version version = VersionUtils.randomVersionBetween(random(), Version.V_5_0_0, + VersionUtils.getPreviousVersion(Version.V_7_0_0_alpha1)); + ClusterState clusterState = createInitialClusterState(true, version); + MetaData.Builder metaBuilder = MetaData.builder(clusterState.metaData()); + metaBuilder.put(IndexMetaData.builder("target").settings(settings(Version.CURRENT) + .put(IndexMetaData.INDEX_RESIZE_SOURCE_NAME.getKey(), "source") + .put(IndexMetaData.INDEX_RESIZE_SOURCE_UUID_KEY, IndexMetaData.INDEX_UUID_NA_VALUE)) + .numberOfShards(4).numberOfReplicas(0)); + MetaData metaData = metaBuilder.build(); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable()); + routingTableBuilder.addAsNew(metaData.index("target")); + + clusterState = ClusterState.builder(clusterState) + .routingTable(routingTableBuilder.build()) + .metaData(metaData).build(); + Index idx = clusterState.metaData().index("target").getIndex(); + + + ResizeAllocationDecider resizeAllocationDecider = new ResizeAllocationDecider(Settings.EMPTY); + RoutingAllocation routingAllocation = new RoutingAllocation(null, clusterState.getRoutingNodes(), clusterState, null, 0); + int shardId = randomIntBetween(0, 3); + int sourceShardId = IndexMetaData.selectSplitShard(shardId, clusterState.metaData().index("source"), 4).id(); + ShardRouting shardRouting = TestShardRouting.newShardRouting(new ShardId(idx, shardId), null, true, RecoverySource + .LocalShardsRecoverySource.INSTANCE, ShardRoutingState.UNASSIGNED); + assertEquals(Decision.YES, resizeAllocationDecider.canAllocate(shardRouting, routingAllocation)); + + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation)); + assertEquals(Decision.NO, resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation)); + + routingAllocation.debugDecision(true); + assertEquals("source primary is active", resizeAllocationDecider.canAllocate(shardRouting, routingAllocation).getExplanation()); + assertEquals("node [node1] is too old to split a shard", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node1"), + routingAllocation).getExplanation()); + assertEquals("node [node2] is too old to split a shard", + resizeAllocationDecider.canAllocate(shardRouting, clusterState.getRoutingNodes().node("node2"), + routingAllocation).getExplanation()); + } } From 31c296afa3c45f43737bbcd6430f414c1f1eb8db Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 17 Oct 2017 11:57:43 +0200 Subject: [PATCH 09/13] add feedback from @bleskes and add master action BWC --- .../admin/indices/shrink/ResizeAction.java | 2 ++ .../admin/indices/shrink/ResizeRequest.java | 6 ++-- .../indices/shrink/TransportResizeAction.java | 21 ++++++++++--- .../indices/shrink/TransportShrinkAction.java | 2 +- .../master/TransportMasterNodeAction.java | 14 ++++++++- .../cluster/metadata/IndexMetaData.java | 8 ++--- .../metadata/MetaDataCreateIndexService.java | 16 +++++----- .../decider/ResizeAllocationDecider.java | 4 +-- .../elasticsearch/index/shard/IndexShard.java | 8 ++--- .../admin/indices/create/SplitIndexIT.java | 31 +++++++++++++------ .../shrink/TransportResizeActionTests.java | 10 +++--- .../MetaDataCreateIndexServiceTests.java | 28 +++++++++-------- 12 files changed, 92 insertions(+), 58 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java index 9bd43b22eec36..9447e0803e2ba 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeAction.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.shrink; +import org.elasticsearch.Version; import org.elasticsearch.action.Action; import org.elasticsearch.client.ElasticsearchClient; @@ -26,6 +27,7 @@ public class ResizeAction extends Action implements IndicesRequest { - public static final ObjectParser PARSER = new ObjectParser<>("shrink_request", null); + public static final ObjectParser PARSER = new ObjectParser<>("resize_request", null); static { PARSER.declareField((parser, request, context) -> request.getTargetIndexRequest().settings(parser.map()), new ParseField("settings"), ObjectParser.ValueType.OBJECT); @@ -84,7 +84,7 @@ public void readFrom(StreamInput in) throws IOException { targetIndexRequest = new CreateIndexRequest(); targetIndexRequest.readFrom(in); sourceIndex = in.readString(); - if (in.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + if (in.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)) { type = in.readEnum(ResizeType.class); } else { type = ResizeType.SHRINK; // BWC this used to be shrink only @@ -96,7 +96,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); targetIndexRequest.writeTo(out); out.writeString(sourceIndex); - if (out.getVersion().onOrAfter(Version.V_7_0_0_alpha1)) { + if (out.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)) { out.writeEnum(type); } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 71751599956dd..ffbea8a22d6f6 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.shrink; import org.apache.lucene.index.IndexWriter; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; @@ -34,6 +35,7 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -94,6 +96,7 @@ protected void masterOperation(final ResizeRequest resizeRequest, final ClusterS // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index()); client.admin().indices().prepareStats(sourceIndex).clear().setDocs(true).execute(new ActionListener() { @Override public void onResponse(IndicesStatsResponse indicesStatsResponse) { @@ -101,7 +104,7 @@ public void onResponse(IndicesStatsResponse indicesStatsResponse) { (i) -> { IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i); return shard == null ? null : shard.getPrimary().getDocs(); - }, indexNameExpressionResolver); + }, sourceIndex, targetIndex); createIndexService.createIndex( updateRequest, ActionListener.wrap(response -> @@ -121,11 +124,9 @@ public void onFailure(Exception e) { // static for unittesting this method static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final ResizeRequest resizeRequest, final ClusterState state - , final IntFunction perShardDocStats, IndexNameExpressionResolver indexNameExpressionResolver) { - final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex()); + , final IntFunction perShardDocStats, String sourceIndexName, String targetIndexName) { final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest(); - final String targetIndexName = indexNameExpressionResolver.resolveDateMathExpression(targetIndex.index()); - final IndexMetaData metaData = state.metaData().index(sourceIndex); + final IndexMetaData metaData = state.metaData().index(sourceIndexName); final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); int numShards = 1; @@ -177,4 +178,14 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi .resizeType(resizeRequest.getResizeType()); } + @Override + protected String getMasterActionName(DiscoveryNode node) { + if (node.getVersion().onOrAfter(ResizeAction.COMPATIBILITY_VERSION)){ + return super.getMasterActionName(node); + } else { + // this is for BWC - when we send this to version that doesn't have ResizeAction.NAME registered + // we have to send to shrink instead. + return ShrinkAction.NAME; + } + } } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java index 9005f083d4736..acc88251970f3 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportShrinkAction.java @@ -32,7 +32,7 @@ /** * Main class to initiate shrinking an index into a new index * This class is only here for backwards compatibility. It will be replaced by - * TransportResizeAction in 8.0 + * TransportResizeAction in 7.x once this is backported */ public class TransportShrinkAction extends TransportResizeAction { diff --git a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java index 8cff2213c2111..feb47aa34fd86 100644 --- a/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java +++ b/core/src/main/java/org/elasticsearch/action/support/master/TransportMasterNodeAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.cluster.NotMasterException; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.io.stream.Writeable; @@ -189,7 +190,10 @@ protected void doRun() throws Exception { logger.debug("no known master node, scheduling a retry"); retry(null, masterChangePredicate); } else { - transportService.sendRequest(nodes.getMasterNode(), actionName, request, new ActionListenerResponseHandler(listener, TransportMasterNodeAction.this::newResponse) { + DiscoveryNode masterNode = nodes.getMasterNode(); + final String actionName = getMasterActionName(masterNode); + transportService.sendRequest(masterNode, actionName, request, new ActionListenerResponseHandler(listener, + TransportMasterNodeAction.this::newResponse) { @Override public void handleException(final TransportException exp) { Throwable cause = exp.unwrapCause(); @@ -229,4 +233,12 @@ public void onTimeout(TimeValue timeout) { ); } } + + /** + * Allows to conditionally return a different master node action name in the case an action gets renamed. + * This mainly for backwards compatibility should be used rarely + */ + protected String getMasterActionName(DiscoveryNode node) { + return actionName; + } } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index b4ed372322989..2a63c8658bc8d 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -456,6 +456,8 @@ public MappingMetaData mapping(String mappingType) { return mappings.get(mappingType); } + // we keep the shrink settings for BWC - this can be removed in 8.0 + // we can't remove in 7 since this setting might be baked into an index coming in via a full cluster restart from 6.0 public static final String INDEX_SHRINK_SOURCE_UUID_KEY = "index.shrink.source.uuid"; public static final String INDEX_SHRINK_SOURCE_NAME_KEY = "index.shrink.source.name"; public static final String INDEX_RESIZE_SOURCE_UUID_KEY = "index.resize.source.uuid"; @@ -468,10 +470,8 @@ public MappingMetaData mapping(String mappingType) { INDEX_SHRINK_SOURCE_NAME); public Index getResizeSourceIndex() { - return INDEX_RESIZE_SOURCE_UUID.exists(settings) || INDEX_SHRINK_SOURCE_UUID.exists(settings) ? new Index - (INDEX_RESIZE_SOURCE_NAME.get - (settings), - INDEX_RESIZE_SOURCE_UUID.get(settings)) : null; + return INDEX_RESIZE_SOURCE_UUID.exists(settings) || INDEX_SHRINK_SOURCE_UUID.exists(settings) + ? new Index(INDEX_RESIZE_SOURCE_NAME.get(settings), INDEX_RESIZE_SOURCE_UUID.get(settings)) : null; } /** diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index d7be87b2f872c..f75788eb3201c 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -612,10 +612,8 @@ static List validateShrinkIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); - - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - IndexMetaData.selectShrinkShards(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); - } + assert IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings); + IndexMetaData.selectShrinkShards(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); if (sourceMetaData.getNumberOfShards() == 1) { throw new IllegalArgumentException("can't shrink an index with only one shard"); @@ -647,10 +645,7 @@ static void validateSplitIndex(ClusterState state, String sourceIndex, Set targetIndexMappingsTypes, String targetIndexName, Settings targetIndexSettings) { IndexMetaData sourceMetaData = validateResize(state, sourceIndex, targetIndexMappingsTypes, targetIndexName, targetIndexSettings); - - if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { - IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); - } + IndexMetaData.selectSplitShard(0, sourceMetaData, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings)); if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { // ensure we have a single type since this would make the splitting code considerably more complex // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity @@ -701,7 +696,10 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma .put(IndexMetaData.INDEX_ROUTING_INITIAL_RECOVERY_GROUP_SETTING.getKey() + "_id", Strings.arrayToCommaDelimitedString(nodesToAllocateOn.toArray())) // we only try once and then give up with a shrink index - .put("index.allocation.max_retries", 1); + .put("index.allocation.max_retries", 1) + // we add the legacy way of specifying it here for BWC. We can remove this once it's backported to 6.x + .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) + .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } else if (type == ResizeType.SPLIT) { validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); } else { diff --git a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java index 40ef62cb0f423..a0ebf7ddba923 100644 --- a/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java +++ b/core/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java @@ -20,6 +20,7 @@ package org.elasticsearch.cluster.routing.allocation.decider; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.shrink.ResizeAction; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.RoutingNode; @@ -36,7 +37,6 @@ /** * An allocation decider that ensures we allocate the shards of a target index for resize operations next to the source primaries - * // TODO add tests!! */ public class ResizeAllocationDecider extends AllocationDecider { @@ -79,7 +79,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing return allocation.decision(Decision.NO, NAME, "source primary shard [%s] is not active", shardId); } if (node != null) { // we might get called from the 2 param canAllocate method.. - if (node.node().getVersion().before(Version.V_7_0_0_alpha1)) { + if (node.node().getVersion().before(ResizeAction.COMPATIBILITY_VERSION)) { return allocation.decision(Decision.NO, NAME, "node [%s] is too old to split a shard", node.nodeId()); } if (sourceShardRouting.currentNodeId().equals(node.nodeId())) { diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3c6295af63309..2f36a88cd563e 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2032,9 +2032,9 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService break; case LOCAL_SHARDS: final IndexMetaData indexMetaData = indexSettings().getIndexMetaData(); - final Index mergeSourceIndex = indexMetaData.getResizeSourceIndex(); + final Index resizeSourceIndex = indexMetaData.getResizeSourceIndex(); final List startedShards = new ArrayList<>(); - final IndexService sourceIndexService = indicesService.indexService(mergeSourceIndex); + final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex); final Set requiredShards; final int numShards; if (sourceIndexService != null) { @@ -2068,9 +2068,9 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService } else { final RuntimeException e; if (numShards == -1) { - e = new IndexNotFoundException(mergeSourceIndex); + e = new IndexNotFoundException(resizeSourceIndex); } else { - e = new IllegalStateException("not all shards from index " + mergeSourceIndex + e = new IllegalStateException("not all shards from index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId()); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index e627bec7b2a96..984b0d2e0a57e 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -58,7 +59,9 @@ import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.IntStream; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -128,12 +131,7 @@ public void testCreateSplitIndexToN() { ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes() .getDataNodes(); assertTrue("at least 2 nodes but was: " + dataNodes.size(), dataNodes.size() >= 2); - DiscoveryNode[] discoveryNodes = dataNodes.values().toArray(DiscoveryNode.class); - // ensure all shards are allocated otherwise the ensure green below might not succeed since we require the merge node - // if we change the setting too quickly we will end up with one replica unassigned which can't be assigned anymore due - // to the require._name below. ensureYellow(); - // relocate all shards to one node such that we can merge it. client().admin().indices().prepareUpdateSettings("source") .setSettings(Settings.builder() .put("index.blocks.write", true)).get(); @@ -162,12 +160,11 @@ public void testCreateSplitIndexToN() { assertTrue(getResponse.isExists()); } - // relocate all shards to one node such that we can merge it. client().admin().indices().prepareUpdateSettings("first_split") .setSettings(Settings.builder() .put("index.blocks.write", true)).get(); ensureGreen(); - // now merge source into a 2 shard index + // now split source into a new index assertAcked(client().admin().indices().prepareResizeIndex("first_split", "second_split") .setResizeType(ResizeType.SPLIT) .setSettings(Settings.builder() @@ -198,6 +195,23 @@ public void testCreateSplitIndexToN() { assertHitCount(client().prepareSearch("second_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); assertHitCount(client().prepareSearch("first_split").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); assertHitCount(client().prepareSearch("source").setSize(100).setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + assertAllUniqueDocs(client().prepareSearch("second_split").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertAllUniqueDocs(client().prepareSearch("first_split").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + assertAllUniqueDocs(client().prepareSearch("source").setSize(100) + .setQuery(new TermsQueryBuilder("foo", "bar")).get(), numDocs); + + } + + public void assertAllUniqueDocs(SearchResponse response, int numDocs) { + Set ids = new HashSet<>(); + for (int i = 0; i < response.getHits().getHits().length; i++) { + String id = response.getHits().getHits()[i].getId(); + assertTrue("found ID "+ id + " more than once", ids.add(id)); + } + assertEquals(numDocs, ids.size()); } public void testSplitIndexPrimaryTerm() throws Exception { @@ -310,7 +324,6 @@ public void testCreateSplitIndex() { )).get(); try { - // now merge source into a single shard index final boolean createWithReplicas = randomBoolean(); assertAcked(client().admin().indices().prepareResizeIndex("source", "target") .setResizeType(ResizeType.SPLIT) @@ -319,7 +332,6 @@ public void testCreateSplitIndex() { .put("index.number_of_shards", 2).build()).get()); ensureGreen(); - // resolve true merge node - this is not always the node we required as all shards may be on another node final ClusterState state = client().admin().cluster().prepareState().get().getState(); DiscoveryNode mergeNode = state.nodes().get(state.getRoutingTable().index("target").shard(0).primaryShard().currentNodeId()); logger.info("split node {}", mergeNode); @@ -410,7 +422,6 @@ public void testCreateSplitWithIndexSort() throws Exception { flushAndRefresh(); assertSortedSegments("source", expectedIndexSort); - // relocate all shards to one node such that we can merge it. client().admin().indices().prepareUpdateSettings("source") .setSettings(Settings.builder() .put("index.blocks.write", true)).get(); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index fd6b73bfc2cad..adbaef6a199e2 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -73,7 +73,7 @@ public void testErrorCondition() { assertTrue( expectThrows(IllegalStateException.class, () -> TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, - (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)) + (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), "target", "source") ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -84,8 +84,7 @@ public void testErrorCondition() { ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); TransportResizeAction.prepareCreateIndexRequest(req, clusterState, - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, - new IndexNameExpressionResolver(Settings.EMPTY)); + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, "target", "source"); } ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -106,7 +105,7 @@ public void testErrorCondition() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, - (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), new IndexNameExpressionResolver(Settings.EMPTY)); + (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), "target", "source"); } public void testShrinkIndexSettings() { @@ -133,8 +132,7 @@ public void testShrinkIndexSettings() { final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( - target, clusterState, (i) -> stats, - new IndexNameExpressionResolver(Settings.EMPTY)); + target, clusterState, (i) -> stats, "target", "source"); assertNotNull(request.recoverFrom()); assertEquals(indexName, request.recoverFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 45525b797ef68..39e4a18440931 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.ResourceAlreadyExistsException; @@ -98,29 +99,28 @@ public void testValidateShrinkIndex() { MetaDataCreateIndexService.validateShrinkIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY) ).getMessage()); + Settings targetSettings = Settings.builder().put("index.number_of_shards", 1).build(); assertEquals("can't shrink an index with only one shard", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", - 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), - "target", Settings.EMPTY) - ).getMessage()); + 1, 0, Settings.builder().put("index.blocks.write", true).build()), "source", + Collections.emptySet(), "target", targetSettings)).getMessage()); assertEquals("the number of target shards [10] must be less that the number of source shards [5]", expectThrows(IllegalArgumentException.class, () -> MetaDataCreateIndexService.validateShrinkIndex(createClusterState("source", - 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", Collections.emptySet(), - "target", Settings.builder().put("index.number_of_shards", 10).build()) - ).getMessage()); + 5, 0, Settings.builder().put("index.blocks.write", true).build()), "source", + Collections.emptySet(), "target", Settings.builder().put("index.number_of_shards", 10).build())).getMessage()); assertEquals("index source must be read-only to resize index. use \"index.blocks.write=true\"", expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateShrinkIndex( createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) - , "source", Collections.emptySet(), "target", Settings.EMPTY) + , "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); assertEquals("index source must have all shards allocated on the same node to shrink index", expectThrows(IllegalStateException.class, () -> - MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", Settings.EMPTY) + MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); assertEquals("the number of source shards [8] must be a must be a multiple of [3]", @@ -133,7 +133,7 @@ public void testValidateShrinkIndex() { assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", expectThrows(IllegalArgumentException.class, () -> { MetaDataCreateIndexService.validateShrinkIndex(state, "source", Collections.singleton("foo"), - "target", Settings.EMPTY); + "target", targetSettings); } ).getMessage()); @@ -161,17 +161,18 @@ public void testValidateShrinkIndex() { public void testValidateSplitIndex() { int numShards = randomIntBetween(1, 42); + Settings targetSettings = Settings.builder().put("index.number_of_shards", numShards * 2).build(); ClusterState state = createClusterState("source", numShards, randomIntBetween(0, 10), Settings.builder().put("index.blocks.write", true).build()); assertEquals("index [source] already exists", expectThrows(ResourceAlreadyExistsException.class, () -> - MetaDataCreateIndexService.validateSplitIndex(state, "target", Collections.emptySet(), "source", Settings.EMPTY) + MetaDataCreateIndexService.validateSplitIndex(state, "target", Collections.emptySet(), "source", targetSettings) ).getMessage()); assertEquals("no such index", expectThrows(IndexNotFoundException.class, () -> - MetaDataCreateIndexService.validateSplitIndex(state, "no such index", Collections.emptySet(), "target", Settings.EMPTY) + MetaDataCreateIndexService.validateSplitIndex(state, "no such index", Collections.emptySet(), "target", targetSettings) ).getMessage()); assertEquals("the number of source shards [10] must be less that the number of target shards [5]", @@ -185,7 +186,7 @@ public void testValidateSplitIndex() { expectThrows(IllegalStateException.class, () -> MetaDataCreateIndexService.validateSplitIndex( createClusterState("source", randomIntBetween(2, 100), randomIntBetween(0, 10), Settings.EMPTY) - , "source", Collections.emptySet(), "target", Settings.EMPTY) + , "source", Collections.emptySet(), "target", targetSettings) ).getMessage()); @@ -199,7 +200,7 @@ public void testValidateSplitIndex() { assertEquals("mappings are not allowed when resizing indices, all mappings are copied from the source index", expectThrows(IllegalArgumentException.class, () -> { MetaDataCreateIndexService.validateSplitIndex(state, "source", Collections.singleton("foo"), - "target", Settings.EMPTY); + "target", targetSettings); } ).getMessage()); @@ -256,6 +257,7 @@ public void testResizeIndexSettings() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); Settings.Builder builder = Settings.builder(); + builder.put("index.number_of_shards", 1); MetaDataCreateIndexService.prepareResizeIndexSettings(clusterState, Collections.emptySet(), builder, clusterState.metaData().index(indexName).getIndex(), "target", ResizeType.SHRINK); assertEquals("similarity settings must be copied", "BM25", builder.build().get("index.similarity.default.type")); From f0175ee350b49b0c1275c81c51bcfbc570d94c96 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 17 Oct 2017 15:27:28 +0200 Subject: [PATCH 10/13] cut over to routing number of shards --- .../indices/shrink/TransportResizeAction.java | 4 +++ .../cluster/metadata/IndexMetaData.java | 20 +++++++++++-- .../metadata/MetaDataCreateIndexService.java | 6 +--- .../common/settings/IndexScopedSettings.java | 2 +- .../common/settings/Setting.java | 6 ++++ .../admin/indices/create/SplitIndexIT.java | 8 ++--- .../shrink/TransportResizeActionTests.java | 8 ++--- .../cluster/metadata/IndexMetaDataTests.java | 22 ++++++++++++++ docs/reference/indices/split-index.asciidoc | 29 ++++++++++++------- .../test/indices.split/10_basic.yml | 2 +- .../test/indices.split/20_source_mapping.yml | 2 +- 11 files changed, 80 insertions(+), 29 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index ffbea8a22d6f6..67ca0cf04bf35 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -39,6 +39,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.threadpool.ThreadPool; @@ -127,6 +128,9 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi , final IntFunction perShardDocStats, String sourceIndexName, String targetIndexName) { final CreateIndexRequest targetIndex = resizeRequest.getTargetIndexRequest(); final IndexMetaData metaData = state.metaData().index(sourceIndexName); + if (metaData == null) { + throw new IndexNotFoundException(sourceIndexName); + } final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); int numShards = 1; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index 2a63c8658bc8d..f6ef3f3a0c2ad 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -65,6 +65,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -195,8 +196,23 @@ static Setting buildNumberOfShardsSetting() { public static final Setting INDEX_ROUTING_PARTITION_SIZE_SETTING = Setting.intSetting(SETTING_ROUTING_PARTITION_SIZE, 1, 1, Property.IndexScope); - public static final Setting INDEX_ROUTING_SHARDS_FACTOR_SETTING = - Setting.intSetting("index.routing_shards_factor", 1, 1, Property.IndexScope); + public static final Setting INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING = + Setting.intSetting("index.number_of_routing_shards", INDEX_NUMBER_OF_SHARDS_SETTING, 1, new Setting.Validator() { + @Override + public void validate(Integer numRoutingShards, Map, Integer> settings) { + Integer numShards = settings.get(INDEX_NUMBER_OF_SHARDS_SETTING); + if (numRoutingShards < numShards) { + throw new IllegalArgumentException("index.number_of_routing_shards [" + numRoutingShards + + "] must be >= index.number_of_shards [" + numShards + "]"); + } + getRoutingFactor(numShards, numRoutingShards); + } + + @Override + public Iterator> settings() { + return Collections.singleton(INDEX_NUMBER_OF_SHARDS_SETTING).iterator(); + } + }, Property.IndexScope); public static final String SETTING_AUTO_EXPAND_REPLICAS = "index.auto_expand_replicas"; public static final Setting INDEX_AUTO_EXPAND_REPLICAS_SETTING = AutoExpandReplicas.SETTING; diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index f75788eb3201c..c3ce57f2f2622 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -384,11 +384,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { final int routingNumShards; if (recoverFromIndex == null) { Settings idxSettings = indexSettingsBuilder.build(); - int numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(idxSettings); - int routingShardsFactor = IndexMetaData.INDEX_ROUTING_SHARDS_FACTOR_SETTING.get(idxSettings); - // we multiply the routing shards in order to split the shard going forward. - // implementation wise splitting shards is really just an inverted shrink operation. - routingNumShards = numShards * routingShardsFactor; + routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); routingNumShards = sourceMetaData.getRoutingNumShards(); diff --git a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index bb120ca31aa24..e575a85c5708d 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/core/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -70,7 +70,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING, IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING, IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING, - IndexMetaData.INDEX_ROUTING_SHARDS_FACTOR_SETTING, + IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING, IndexMetaData.INDEX_READ_ONLY_SETTING, IndexMetaData.INDEX_BLOCKS_READ_SETTING, IndexMetaData.INDEX_BLOCKS_WRITE_SETTING, diff --git a/core/src/main/java/org/elasticsearch/common/settings/Setting.java b/core/src/main/java/org/elasticsearch/common/settings/Setting.java index 9c70c35ad8bca..9b99e67c8c4da 100644 --- a/core/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/core/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -908,6 +908,12 @@ public static Setting intSetting(String key, Setting fallbackS return new Setting<>(key, fallbackSetting, (s) -> parseInt(s, minValue, key), properties); } + public static Setting intSetting(String key, Setting fallbackSetting, int minValue, Validator validator, + Property... properties) { + return new Setting<>(new SimpleKey(key), fallbackSetting, fallbackSetting::getRaw, (s) -> parseInt(s, minValue, key),validator, + properties); + } + public static Setting longSetting(String key, long defaultValue, long minValue, Property... properties) { return new Setting<>(key, (s) -> Long.toString(defaultValue), (s) -> parseLong(s, minValue, key), properties); } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 984b0d2e0a57e..96a5f5cb872f6 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -89,7 +89,7 @@ public void testCreateSplitIndexToN() { CreateIndexRequestBuilder createInitialIndex = prepareCreate("source"); Settings.Builder settings = Settings.builder().put(indexSettings()) .put("number_of_shards", shardSplits[0]) - .put("index.routing_shards_factor", shardSplits[2] / shardSplits[0]); + .put("index.number_of_routing_shards", shardSplits[2] * randomIntBetween(1, 10)); if (useRouting && useMixedRouting == false && randomBoolean()) { settings.put("index.routing_partition_size", randomIntBetween(1, 10)); createInitialIndex.addMapping("t1", "_routing", "required=true"); @@ -222,7 +222,7 @@ public void testSplitIndexPrimaryTerm() throws Exception { internalCluster().ensureAtLeastNumDataNodes(2); prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) .put("number_of_shards", numberOfShards) - .put("index.routing_shards_factor", numberOfTargetShards / numberOfShards)).get(); + .put("index.number_of_routing_shards", numberOfTargetShards)).get(); final ImmutableOpenMap dataNodes = client().admin().cluster().prepareState().get().getState().nodes().getDataNodes(); @@ -295,7 +295,7 @@ public void testCreateSplitIndex() { prepareCreate("source").setSettings(Settings.builder().put(indexSettings()) .put("number_of_shards", 1) .put("index.version.created", version) - .put("index.routing_shards_factor", 2) + .put("index.number_of_routing_shards", 2) ).get(); final int docs = randomIntBetween(0, 128); for (int i = 0; i < docs; i++) { @@ -398,7 +398,7 @@ public void testCreateSplitWithIndexSort() throws Exception { Settings.builder() .put(indexSettings()) .put("sort.field", "id") - .put("index.routing_shards_factor", 16) + .put("index.number_of_routing_shards", 16) .put("sort.order", "desc") .put("number_of_shards", 2) .put("number_of_replicas", 0) diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index adbaef6a199e2..d0f76f9a9462a 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -73,7 +73,7 @@ public void testErrorCondition() { assertTrue( expectThrows(IllegalStateException.class, () -> TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, - (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), "target", "source") + (i) -> new DocsStats(Integer.MAX_VALUE, randomIntBetween(1, 1000)), "source", "target") ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -84,7 +84,7 @@ public void testErrorCondition() { ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); TransportResizeAction.prepareCreateIndexRequest(req, clusterState, - (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, "target", "source"); + (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE/2, randomIntBetween(1, 1000)) : null, "source", "target"); } ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -105,7 +105,7 @@ public void testErrorCondition() { clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, - (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), "target", "source"); + (i) -> new DocsStats(randomIntBetween(1, 1000), randomIntBetween(1, 1000)), "source", "target"); } public void testShrinkIndexSettings() { @@ -132,7 +132,7 @@ public void testShrinkIndexSettings() { final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE; target.setWaitForActiveShards(activeShardCount); CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest( - target, clusterState, (i) -> stats, "target", "source"); + target, clusterState, (i) -> stats, indexName, "target"); assertNotNull(request.recoverFrom()); assertEquals(indexName, request.recoverFrom().getName()); assertEquals("1", request.settings().get("index.number_of_shards")); diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index 85ba918068eba..bb63f36d3307a 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -202,4 +202,26 @@ public void testIndexFormat() { assertThat(metaData.getSettings().getAsInt(IndexMetaData.INDEX_FORMAT_SETTING.getKey(), 0), is(0)); } } + + public void testNumberOfRoutingShards() { + Settings build = Settings.builder().put("index.number_of_shards", 5).put("index.number_of_routing_shards", 10).build(); + assertEquals(10, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + build = Settings.builder().put("index.number_of_shards", 5).put("index.number_of_routing_shards", 5).build(); + assertEquals(5, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + int numShards = randomIntBetween(1, 10); + build = Settings.builder().put("index.number_of_shards", numShards).build(); + assertEquals(numShards, IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(build).intValue()); + + Settings lessThanSettings = Settings.builder().put("index.number_of_shards", 8).put("index.number_of_routing_shards", 4).build(); + IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, + () -> IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(lessThanSettings)); + assertEquals("index.number_of_routing_shards [4] must be >= index.number_of_shards [8]", iae.getMessage()); + + Settings notAFactorySettings = Settings.builder().put("index.number_of_shards", 2).put("index.number_of_routing_shards", 3).build(); + iae = expectThrows(IllegalArgumentException.class, + () -> IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(notAFactorySettings)); + assertEquals("the number of source shards [2] must be a must be a factor of [3]", iae.getMessage()); + } } diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index bf1434f525d80..5464fd801262f 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -1,15 +1,23 @@ [[indices-split-index]] == Split Index +number_of_routing_shards + The split index API allows you to split an existing index into a new index with multiple of it's primary shards. Similarly to the <> where the number of primary shards in the shrunk index must be a factor of the source index. -The `_split` API requires the source index to be created with a routing shard factor in order -to be split. (Note: this requirement might be remove in future releases) For example an index -with `8` primary shards and a `index.routing_shards_factor` of `2` can be split into `16` -primary shards or an index with `1` primary shard and `index.routing_shards_factor` of `64` -can be split into `2`, `4`, `8`, `16`, `32` or `64`. -Before splitting, a (primary) copy of every shard in the index must be active in the cluster. +The `_split` API requires the source index to be created with a specific number of routing shards +in order to be split in the future. (Note: this requirement might be remove in future releases) +The number of routing shards specify the hashing space that is used internally to distribute documents +across shards, in oder to have a consistent hashing that is compatible with the method elasticsearch +uses today. +For example an index with `8` primary shards and a `index.number_of_routing_shards` of `32` +can be split into `16` and `32` primary shards. An index with `1` primary shard +and `index.number_of_routing_shards` of `64` can be split into `2`, `4`, `8`, `16`, `32` or `64`. +The same works for non power of two routing shards ie. an index with `1` primary shard and +`index.number_of_routing_shards` set to `15` can be split into `3` and `15` or alternatively`5` and `15`. +The number of shards in the split index must always be a factor of `index.number_of_routing_shards` +in the source index. Before splitting, a (primary) copy of every shard in the index must be active in the cluster. Splitting works as follows: @@ -37,15 +45,14 @@ PUT my_source_index { "settings": { "index.number_of_shards" : 1, - "index.routing_shards_factor" : 2 <1> + "index.number_of_routing_shards" : 2 <1> } } ------------------------------------------------- // CONSOLE -<1> Allows to split the index into twice as many shards ie. it allows - for a single split operation. The split will allow to go from the - default of 5 shards to 10 shards. +<1> Allows to split the index into two shards or in other words, it allows + for a single split operation. In order to split an index, the index must be marked as read-only, and have <> `green`. @@ -121,7 +128,7 @@ POST my_source_index/_split/my_target_index } -------------------------------------------------- // CONSOLE -// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.routing_shards_factor" : 5, "index.number_of_shards": "1"}}\n/] +// TEST[s/^/PUT my_source_index\n{"settings": {"index.blocks.write": true, "index.number_of_routing_shards" : 5, "index.number_of_shards": "1"}}\n/] <1> The number of shards in the target index. This must be a factor of the number of shards in the source index. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml index 6ce39e311853a..f51fc808b4623 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml @@ -11,7 +11,7 @@ settings: index.number_of_shards: 1 index.number_of_replicas: 0 - index.routing_shards_factor: 2 + index.number_of_routing_shards: 2 - do: index: index: source diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml index 790f7d6d18402..ffd7ffe7a2946 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/20_source_mapping.yml @@ -13,7 +13,7 @@ settings: number_of_shards: 1 number_of_replicas: 0 - index.routing_shards_factor: 2 + index.number_of_routing_shards: 2 mappings: test: properties: From 5c1a58a3783ab831a87ec2932d19088a8b4292ae Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 1 Nov 2017 12:51:31 +0100 Subject: [PATCH 11/13] apply feedback from @jpoutz --- .../org/elasticsearch/index/shard/ShardSplittingQuery.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java index 04924f5113922..94aee085175a0 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/ShardSplittingQuery.java @@ -85,12 +85,12 @@ public Scorer scorer(LeafReaderContext context) throws IOException { return shardId == targetShardId; }; if (terms == null) { // this is the common case - no partitioning and no _routing values + assert indexMetaData.isRoutingPartitionedIndex() == false; findSplitDocs(IdFieldMapper.NAME, includeInShard, leafReader, bitSet::set); } else { if (indexMetaData.isRoutingPartitionedIndex()) { // this is the heaviest invariant. Here we have to visit all docs stored fields do extract _id and _routing // this this index is routing partitioned. - Bits liveDocs = leafReader.getLiveDocs(); Visitor visitor = new Visitor(); return new ConstantScoreScorer(this, score(), new RoutingPartitionedDocIdSetIterator(leafReader, visitor)); @@ -142,7 +142,7 @@ public boolean equals(Object o) { public int hashCode() { int result = indexMetaData.hashCode(); result = 31 * result + shardId; - return result; + return classHash() ^ result; } private static void findSplitDocs(String idField, Predicate includeInShard, From 264ae4c5e2fe0577746ed56d591fbb33a9618c1b Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 1 Nov 2017 14:53:10 +0100 Subject: [PATCH 12/13] apply feedback from @bleskes --- .../action/admin/indices/shrink/ResizeRequest.java | 4 ++++ .../admin/indices/shrink/TransportResizeAction.java | 10 ++++++++-- .../elasticsearch/cluster/metadata/IndexMetaData.java | 3 ++- .../cluster/metadata/MetaDataCreateIndexService.java | 2 ++ .../java/org/elasticsearch/index/shard/IndexShard.java | 2 +- .../org/elasticsearch/index/shard/StoreRecovery.java | 10 +++++----- .../action/admin/indices/create/SplitIndexIT.java | 3 +-- .../indices/shrink/TransportResizeActionTests.java | 7 +++---- .../cluster/metadata/IndexMetaDataTests.java | 4 ++++ docs/reference/indices/split-index.asciidoc | 4 ++-- 10 files changed, 32 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java index c3c950c7f3eaf..f2f648f70ffa9 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/ResizeRequest.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -71,6 +72,9 @@ public ActionRequestValidationException validate() { if (targetIndexRequest.settings().getByPrefix("index.sort.").isEmpty() == false) { validationException = addValidationError("can't override index sort when resizing an index", validationException); } + if (type == ResizeType.SPLIT && IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexRequest.settings()) == false) { + validationException = addValidationError("index.number_of_shards is required for split operations", validationException); + } return validationException; } diff --git a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java index 67ca0cf04bf35..87dd9f9fa2d21 100644 --- a/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/core/src/main/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeAction.java @@ -133,9 +133,12 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi } final Settings targetIndexSettings = Settings.builder().put(targetIndex.settings()) .normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build(); - int numShards = 1; + final int numShards; if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) { numShards = IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings); + } else { + assert resizeRequest.getResizeType() == ResizeType.SHRINK : "split must specify the number of shards explicitly"; + numShards = 1; } for (int i = 0; i < numShards; i++) { @@ -161,6 +164,9 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi if (IndexMetaData.INDEX_ROUTING_PARTITION_SIZE_SETTING.exists(targetIndexSettings)) { throw new IllegalArgumentException("cannot provide a routing partition size value when resizing an index"); } + if (IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(targetIndexSettings)) { + throw new IllegalArgumentException("cannot provide index.number_of_routing_shards on resize"); + } String cause = resizeRequest.getResizeType().name().toLowerCase(Locale.ROOT) + "_index"; targetIndex.cause(cause); Settings.Builder settingsBuilder = Settings.builder().put(targetIndexSettings); @@ -169,7 +175,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(final Resi return new CreateIndexClusterStateUpdateRequest(targetIndex, cause, targetIndex.index(), targetIndexName, true) - // mappings are updated on the node when merging in the shards, this prevents race-conditions since all mapping must be + // mappings are updated on the node when creating in the shards, this prevents race-conditions since all mapping must be // applied once we took the snapshot and if somebody messes things up and switches the index read/write and adds docs we miss // the mappings for everything is corrupted and hard to debug .ackTimeout(targetIndex.timeout()) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java index f6ef3f3a0c2ad..3d14670e52771 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java @@ -1341,9 +1341,10 @@ public static ShardId selectSplitShard(int shardId, IndexMetaData sourceIndexMet if (numSourceShards > numTargetShards) { throw new IllegalArgumentException("the number of source shards [" + numSourceShards + "] must be less that the number of target shards [" + numTargetShards + "]"); - } int routingFactor = getRoutingFactor(numSourceShards, numTargetShards); + // this is just an additional assertion that ensures we are a factor of the routing num shards. + assert getRoutingFactor(numTargetShards, sourceIndexMetadata.getRoutingNumShards()) >= 0; return new ShardId(sourceIndexMetadata.getIndex(), shardId/routingFactor); } diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index c3ce57f2f2622..e22a7d658e894 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -645,6 +645,7 @@ static void validateSplitIndex(ClusterState state, String sourceIndex, if (sourceMetaData.getCreationVersion().before(Version.V_6_0_0_alpha1)) { // ensure we have a single type since this would make the splitting code considerably more complex // and a 5.x index would not be splittable unless it has been shrunk before so rather opt out of the complexity + // since in 5.x we don't have a setting to artificially set the number of routing shards throw new IllegalStateException("source index created version is too old to apply a split operation"); } @@ -697,6 +698,7 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } else if (type == ResizeType.SPLIT) { + validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); } else { throw new IllegalStateException("unknown resize type is " + type); diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java index db392831d80cc..8e04243ce6d96 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -2079,7 +2079,7 @@ public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService if (numShards == -1) { e = new IndexNotFoundException(resizeSourceIndex); } else { - e = new IllegalStateException("not all shards from index " + resizeSourceIndex + e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId()); } diff --git a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java index cfd9972c55a05..b59ab14961769 100644 --- a/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java +++ b/core/src/main/java/org/elasticsearch/index/shard/StoreRecovery.java @@ -108,15 +108,15 @@ boolean recoverFromLocalShards(BiConsumer mappingUpdate if (indices.size() > 1) { throw new IllegalArgumentException("can't add shards from more than one index"); } - IndexMetaData indexMetaData = shards.get(0).getIndexMetaData(); - for (ObjectObjectCursor mapping : indexMetaData.getMappings()) { + IndexMetaData sourceMetaData = shards.get(0).getIndexMetaData(); + for (ObjectObjectCursor mapping : sourceMetaData.getMappings()) { mappingUpdateConsumer.accept(mapping.key, mapping.value); } - indexShard.mapperService().merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); + indexShard.mapperService().merge(sourceMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true); // now that the mapping is merged we can validate the index sort configuration. Sort indexSort = indexShard.getIndexSort(); - final boolean isSplit = indexMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); - assert isSplit == false || indexMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + + final boolean isSplit = sourceMetaData.getNumberOfShards() < indexShard.indexSettings().getNumberOfShards(); + assert isSplit == false || sourceMetaData.getCreationVersion().onOrAfter(Version.V_6_0_0_alpha1) : "for split we require a " + "single type but the index is created before 6.0.0"; return executeRecovery(indexShard, () -> { logger.debug("starting recovery from local shards {}", shards); diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java index 96a5f5cb872f6..8f24edf8577e4 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/create/SplitIndexIT.java @@ -105,8 +105,7 @@ public void testCreateSplitIndexToN() { if (useRouting) { String routing = randomRealisticUnicodeOfCodepointLengthBetween(1, 10); if (useMixedRouting && randomBoolean()) { - routingValue[i] = routing; - + routingValue[i] = null; } else { routingValue[i] = routing; } diff --git a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java index 952d9b57111bf..b03b043f03e14 100644 --- a/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java +++ b/core/src/test/java/org/elasticsearch/action/admin/indices/shrink/TransportResizeActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.cluster.EmptyClusterInfoService; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -72,7 +71,7 @@ public void testErrorCondition() { Settings.builder().put("index.blocks.write", true).build()); assertTrue( expectThrows(IllegalStateException.class, () -> - TransportShrinkAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), state, (i) -> new DocsStats(Integer.MAX_VALUE, between(1, 1000), between(1, 100)), "source", "target") ).getMessage().startsWith("Can't merge index with more than [2147483519] docs - too many documents in shards ")); @@ -83,7 +82,7 @@ public void testErrorCondition() { req.getTargetIndexRequest().settings(Settings.builder().put("index.number_of_shards", 4)); ClusterState clusterState = createClusterState("source", 8, 1, Settings.builder().put("index.blocks.write", true).build()); - TransportShrinkAction.prepareCreateIndexRequest(req, clusterState, + TransportResizeAction.prepareCreateIndexRequest(req, clusterState, (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null , "source", "target"); } @@ -105,7 +104,7 @@ public void testErrorCondition() { routingTable.index("source").shardsWithState(ShardRoutingState.INITIALIZING)).routingTable(); clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build(); - TransportShrinkAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, + TransportResizeAction.prepareCreateIndexRequest(new ResizeRequest("target", "source"), clusterState, (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)), "source", "target"); } diff --git a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java index bb63f36d3307a..e83d1fa706cfd 100644 --- a/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java +++ b/core/src/test/java/org/elasticsearch/cluster/metadata/IndexMetaDataTests.java @@ -157,6 +157,7 @@ public void testSelectSplitShard() { .put("index.number_of_replicas", 0) .build()) .creationDate(randomLong()) + .setRoutingNumShards(4) .build(); ShardId shardId = IndexMetaData.selectSplitShard(0, metaData, 4); assertEquals(0, shardId.getId()); @@ -169,6 +170,9 @@ public void testSelectSplitShard() { assertEquals("the number of target shards (0) must be greater than the shard id: 0", expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 0)).getMessage()); + + assertEquals("the number of source shards [2] must be a must be a factor of [3]", + expectThrows(IllegalArgumentException.class, () -> IndexMetaData.selectSplitShard(0, metaData, 3)).getMessage()); } public void testIndexFormat() { diff --git a/docs/reference/indices/split-index.asciidoc b/docs/reference/indices/split-index.asciidoc index 5464fd801262f..467c09baa2432 100644 --- a/docs/reference/indices/split-index.asciidoc +++ b/docs/reference/indices/split-index.asciidoc @@ -104,8 +104,8 @@ Indices can only be split if they satisfy the following requirements: * The index must have less primary shards than the target index. -* The number of primary shards in the target index must be a power of two - factor of the number of primary shards in the source index. +* The number of primary shards in the target index must be a factor of the + number of primary shards in the source index. * The node handling the split process must have sufficient free disk space to accommodate a second copy of the existing index. From b61c79c76b0f76aeedfc80ffb19e49f3400373cc Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Thu, 2 Nov 2017 15:20:27 +0100 Subject: [PATCH 13/13] add extra assertion --- .../cluster/metadata/MetaDataCreateIndexService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index e22a7d658e894..49568ab300f03 100644 --- a/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -386,9 +386,13 @@ public ClusterState execute(ClusterState currentState) throws Exception { Settings idxSettings = indexSettingsBuilder.build(); routingNumShards = IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.get(idxSettings); } else { + assert IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.exists(indexSettingsBuilder.build()) == false + : "index.number_of_routing_shards should be present on the target index on resize"; final IndexMetaData sourceMetaData = currentState.metaData().getIndexSafe(recoverFromIndex); routingNumShards = sourceMetaData.getRoutingNumShards(); } + // remove the setting it's temporary and is only relevant once we create the index + indexSettingsBuilder.remove(IndexMetaData.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING.getKey()); tmpImdBuilder.setRoutingNumShards(routingNumShards); if (recoverFromIndex != null) { @@ -698,7 +702,6 @@ static void prepareResizeIndexSettings(ClusterState currentState, Set ma .put(IndexMetaData.INDEX_SHRINK_SOURCE_NAME.getKey(), resizeSourceIndex.getName()) .put(IndexMetaData.INDEX_SHRINK_SOURCE_UUID.getKey(), resizeSourceIndex.getUUID()); } else if (type == ResizeType.SPLIT) { - validateSplitIndex(currentState, resizeSourceIndex.getName(), mappingKeys, resizeIntoName, indexSettingsBuilder.build()); } else { throw new IllegalStateException("unknown resize type is " + type);