From cbec51adc26cd55ef6b5dc1269ac6c16f9570313 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Jul 2021 12:04:16 +0200 Subject: [PATCH 1/8] [Rest Api Compatibility] Deprecate the use of synced flush synced flush is going to be replaced by flush. This commit deprecates the synced flush and is meant to be backported to 7.x relates #50882 relates #51816 --- rest-api-spec/build.gradle | 11 ++++++--- .../admin/indices/RestSyncedFlushAction.java | 23 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 1a7251640334a..ceb6dec7e4d6a 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -88,9 +88,7 @@ tasks.named("yamlRestCompatTest").configure { OS.current() != OS.WINDOWS } systemProperty 'tests.rest.blacklist', ([ - 'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion and specifying both node_ids and node_names', - 'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion without specifying nodes', - 'indices.flush/10_basic/Index synced flush rest test', +// 'indices.flush/10_basic/Index synced flush rest test', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', // not fixing this in #70966 'indices.put_template/11_basic_with_types/Put template with empty mappings', @@ -227,6 +225,13 @@ tasks.named("transformV7RestTests").configure({ task -> task.removeWarningForTest("the default value for the ?wait_for_active_shards parameter will change from '0' to 'index-setting' in version 8; " + "specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour" , "?wait_for_active_shards default is deprecated") + + // override for exception message change in #55291 tests cluster.voting_config_exclusions/10_basic/ + // 'Throw exception when adding voting config exclusion and specifying both node_ids and node_names', + // 'Throw exception when adding voting config exclusion without specifying nodes', + task.replaceValueTextByKeyValue("catch", + '/Please set node identifiers correctly. One and only one of \\[node_name\\], \\[node_names\\] and \\[node_ids\\] has to be set/', + '/You must set \\[node_names\\] or \\[node_ids\\] but not both/') }) tasks.register('enforceYamlTestConvention').configure { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java index e750ec093b541..1af97bc12a0ab 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; @@ -32,15 +33,23 @@ public class RestSyncedFlushAction extends BaseRestHandler { - private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RestSyncedFlushAction.class); - + private static final String DEPRECATION_MESSAGE = + "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead."; @Override public List routes() { return List.of( - new Route(GET, "/_flush/synced"), - new Route(POST, "/_flush/synced"), - new Route(GET, "/{index}/_flush/synced"), - new Route(POST, "/{index}/_flush/synced")); + Route.builder(GET, "/_flush/synced") + .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build(), + Route.builder(POST, "/_flush/synced") + .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build(), + Route.builder(GET, "/{index}/_flush/synced") + .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build(), + Route.builder(POST, "/{index}/_flush/synced") + .deprecated(DEPRECATION_MESSAGE, RestApiVersion.V_7) + .build()); } @Override @@ -50,8 +59,6 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - DEPRECATION_LOGGER.deprecate(DeprecationCategory.API, "synced_flush", - "Synced flush was removed and a normal flush was performed instead. This transition will be removed in a future version."); final FlushRequest flushRequest = new FlushRequest(Strings.splitStringByCommaToArray(request.param("index"))); flushRequest.indicesOptions(IndicesOptions.fromRequest(request, flushRequest.indicesOptions())); return channel -> client.admin().indices().flush(flushRequest, new SimulateSyncedFlushResponseListener(channel)); From 75c87227ca1f865c0d82151714e5de3bf011e2da Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Jul 2021 13:41:03 +0200 Subject: [PATCH 2/8] test transformations --- .../java/org/elasticsearch/upgrades/FullClusterRestartIT.java | 4 +--- rest-api-spec/build.gradle | 4 ++++ .../rest/action/admin/indices/RestSyncedFlushAction.java | 4 +--- .../elasticsearch/integration/IndexPrivilegeIntegTests.java | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index 1c4c972b83347..fad6772941075 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -682,9 +682,7 @@ public void testRecovery() throws Exception { flushRequest.addParameter("force", "true"); flushRequest.addParameter("wait_if_ongoing", "true"); assertOK(client().performRequest(flushRequest)); - if (randomBoolean()) { - syncedFlush(index); - } + if (shouldHaveTranslog) { // Update a few documents so we are sure to have a translog diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index ceb6dec7e4d6a..f276d0107b6f7 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -232,6 +232,10 @@ tasks.named("transformV7RestTests").configure({ task -> task.replaceValueTextByKeyValue("catch", '/Please set node identifiers correctly. One and only one of \\[node_name\\], \\[node_names\\] and \\[node_ids\\] has to be set/', '/You must set \\[node_names\\] or \\[node_ids\\] but not both/') + + // sync_id is no longer available in SegmentInfos from lucene // "indices.flush/10_basic/Index synced flush rest test" + task.replaceIsTrue("indices.testing.shards.0.0.commit.user_data.sync_id", "indices.testing.shards.0.0.commit.user_data") + }) tasks.register('enforceYamlTestConvention').configure { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java index 1af97bc12a0ab..a2bd549f32c22 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestSyncedFlushAction.java @@ -13,8 +13,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.DeprecationCategory; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.rest.BaseRestHandler; @@ -34,7 +32,7 @@ public class RestSyncedFlushAction extends BaseRestHandler { private static final String DEPRECATION_MESSAGE = - "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead."; + "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead."; @Override public List routes() { return List.of( diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java index 3204c0f41eb34..c7471ceb40a21 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/IndexPrivilegeIntegTests.java @@ -472,12 +472,10 @@ private void assertUserExecutes(String user, String action, String index, boolea if (userIsAllowed) { assertAccessIsAllowed(user, "POST", "/" + index + "/_refresh"); assertAccessIsAllowed(user, "POST", "/" + index + "/_flush"); - assertAccessIsAllowed(user, "POST", "/" + index + "/_flush/synced"); assertAccessIsAllowed(user, "POST", "/" + index + "/_forcemerge"); } else { assertAccessIsDenied(user, "POST", "/" + index + "/_refresh"); assertAccessIsDenied(user, "POST", "/" + index + "/_flush"); - assertAccessIsDenied(user, "POST", "/" + index + "/_flush/synced"); assertAccessIsDenied(user, "POST", "/" + index + "/_forcemerge"); } break; From a1b0754e3f6896a057a38a298e59dca74efcbe29 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Jul 2021 14:51:37 +0200 Subject: [PATCH 3/8] comment and bwc test --- .../org/elasticsearch/backwards/IndexingIT.java | 15 ++++++++++++--- rest-api-spec/build.gradle | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 0893f7138783c..4beea5b1d6e90 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -17,8 +17,11 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.MediaType; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.rest.ESRestTestCase; @@ -309,9 +312,15 @@ public void testSyncedFlushTransition() throws Exception { try (RestClient newNodeClient = buildClient(restClientSettings(), nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { Request request = new Request("POST", index + "/_flush/synced"); - List warningMsg = List.of("Synced flush was removed and a normal flush was performed instead. " + - "This transition will be removed in a future version."); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warnings -> warnings.equals(warningMsg) == false)); + final String v7MediaType = XContentType.VND_JSON.toParsedMediaType() + .responseContentTypeHeader(Map.of(MediaType.COMPATIBLE_WITH_PARAMETER_NAME, + String.valueOf(RestApiVersion.minimumSupported().major))); + List warningMsg = List.of("Synced flush is deprecated and will be removed in 8.0." + + " Use flush at /_flush or /{index}/_flush instead."); + request.setOptions(RequestOptions.DEFAULT.toBuilder() + .setWarningsHandler(warnings -> warnings.equals(warningMsg) == false) + .addHeader("Accept", v7MediaType)); + assertBusy(() -> { Map result = ObjectPath.createFromResponse(newNodeClient.performRequest(request)).evaluate("_shards"); assertThat(result.get("total"), equalTo(totalShards)); diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index f276d0107b6f7..f545b87a383b9 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -233,7 +233,7 @@ tasks.named("transformV7RestTests").configure({ task -> '/Please set node identifiers correctly. One and only one of \\[node_name\\], \\[node_names\\] and \\[node_ids\\] has to be set/', '/You must set \\[node_names\\] or \\[node_ids\\] but not both/') - // sync_id is no longer available in SegmentInfos from lucene // "indices.flush/10_basic/Index synced flush rest test" + // sync_id is no longer available in SegmentInfos.userData // "indices.flush/10_basic/Index synced flush rest test" task.replaceIsTrue("indices.testing.shards.0.0.commit.user_data.sync_id", "indices.testing.shards.0.0.commit.user_data") }) From 0260c5694842032dc30b381600931754bd3834fb Mon Sep 17 00:00:00 2001 From: pgomulka Date: Thu, 15 Jul 2021 15:39:56 +0200 Subject: [PATCH 4/8] remove the use of synced flush --- .../upgrades/FullClusterRestartIT.java | 4 +- .../elasticsearch/upgrades/RecoveryIT.java | 6 +-- .../test/rest/ESRestTestCase.java | 42 ------------------- 3 files changed, 6 insertions(+), 46 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index fad6772941075..b13f5a0303fa4 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -682,7 +682,9 @@ public void testRecovery() throws Exception { flushRequest.addParameter("force", "true"); flushRequest.addParameter("wait_if_ongoing", "true"); assertOK(client().performRequest(flushRequest)); - + if (randomBoolean()) { + flush(index, randomBoolean()); + } if (shouldHaveTranslog) { // Update a few documents so we are sure to have a translog diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java index a8f04931ea34e..41996f2b9e434 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java @@ -271,7 +271,7 @@ public void testRelocationWithConcurrentIndexing() throws Exception { throw new IllegalStateException("unknown type " + CLUSTER_TYPE); } if (randomBoolean()) { - syncedFlush(index); + flush(index, randomBoolean()); } } @@ -309,7 +309,7 @@ public void testRecovery() throws Exception { } } if (randomBoolean()) { - syncedFlush(index); + flush(index, randomBoolean()); } ensureGreen(index); } @@ -584,7 +584,7 @@ public void testUpdateDoc() throws Exception { assertThat(XContentMapValues.extractValue("_source.updated_field", doc), equalTo(updates.get(docId))); } if (randomBoolean()) { - syncedFlush(index); + flush(index, randomBoolean()); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index ae5057787ad41..5c34cc3ab6b2f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1650,48 +1650,6 @@ protected static Version minimumNodeVersion() throws IOException { return minVersion; } - protected void syncedFlush(String indexName) throws Exception { - final List deprecationMessages = List.of( - "Synced flush is deprecated and will be removed in 8.0. Use flush at _/flush or /{index}/_flush instead."); - final List fixedDeprecationMessages = List.of( - "Synced flush is deprecated and will be removed in 8.0. Use flush at /_flush or /{index}/_flush instead."); - final List transitionMessages = List.of( - "Synced flush was removed and a normal flush was performed instead. This transition will be removed in a future version."); - final WarningsHandler warningsHandler; - if (minimumNodeVersion().onOrAfter(Version.V_8_0_0)) { - warningsHandler = warnings -> warnings.equals(transitionMessages) == false; - } else if (minimumNodeVersion().onOrAfter(Version.V_7_6_0)) { - warningsHandler = warnings -> warnings.equals(deprecationMessages) == false && warnings.equals(transitionMessages) == false && - warnings.equals(fixedDeprecationMessages) == false; - } else if (nodeVersions.stream().anyMatch(n -> n.onOrAfter(Version.V_8_0_0))) { - warningsHandler = warnings -> warnings.isEmpty() == false && warnings.equals(transitionMessages) == false; - } else { - warningsHandler = warnings -> warnings.isEmpty() == false; - } - // We have to spin synced-flush requests here because we fire the global checkpoint sync for the last write operation. - // A synced-flush request considers the global checkpoint sync as an going operation because it acquires a shard permit. - assertBusy(() -> { - try { - final Request request = new Request("POST", indexName + "/_flush/synced"); - request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(warningsHandler)); - Response resp = client().performRequest(request); - if (nodeVersions.stream().allMatch(v -> v.before(Version.V_8_0_0))) { - Map result = ObjectPath.createFromResponse(resp).evaluate("_shards"); - assertThat(result.get("failed"), equalTo(0)); - } - } catch (ResponseException ex) { - if (ex.getResponse().getStatusLine().getStatusCode() == RestStatus.CONFLICT.getStatus() - && ex.getResponse().getWarnings().equals(transitionMessages)) { - logger.info("a normal flush was performed instead"); - } else { - throw new AssertionError(ex); // cause assert busy to retry - } - } - }); - // ensure the global checkpoint is synced; otherwise we might trim the commit with syncId - ensureGlobalCheckpointSynced(indexName); - } - @SuppressWarnings("unchecked") private void ensureGlobalCheckpointSynced(String index) throws Exception { assertBusy(() -> { From bad7fa029bec1e426ae38d34fdc70e668f362d2e Mon Sep 17 00:00:00 2001 From: pgomulka Date: Fri, 16 Jul 2021 09:21:02 +0200 Subject: [PATCH 5/8] remove comment and remove changes related to voting config exclusions --- rest-api-spec/build.gradle | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index f545b87a383b9..ba5944f1c6518 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -88,7 +88,8 @@ tasks.named("yamlRestCompatTest").configure { OS.current() != OS.WINDOWS } systemProperty 'tests.rest.blacklist', ([ -// 'indices.flush/10_basic/Index synced flush rest test', + 'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion and specifying both node_ids and node_names', + 'cluster.voting_config_exclusions/10_basic/Throw exception when adding voting config exclusion without specifying nodes', 'indices.forcemerge/10_basic/Check deprecation warning when incompatible only_expunge_deletes and max_num_segments values are both set', // not fixing this in #70966 'indices.put_template/11_basic_with_types/Put template with empty mappings', @@ -226,13 +227,6 @@ tasks.named("transformV7RestTests").configure({ task -> "specify '?wait_for_active_shards=index-setting' to adopt the future default behaviour, or '?wait_for_active_shards=0' to preserve today's behaviour" , "?wait_for_active_shards default is deprecated") - // override for exception message change in #55291 tests cluster.voting_config_exclusions/10_basic/ - // 'Throw exception when adding voting config exclusion and specifying both node_ids and node_names', - // 'Throw exception when adding voting config exclusion without specifying nodes', - task.replaceValueTextByKeyValue("catch", - '/Please set node identifiers correctly. One and only one of \\[node_name\\], \\[node_names\\] and \\[node_ids\\] has to be set/', - '/You must set \\[node_names\\] or \\[node_ids\\] but not both/') - // sync_id is no longer available in SegmentInfos.userData // "indices.flush/10_basic/Index synced flush rest test" task.replaceIsTrue("indices.testing.shards.0.0.commit.user_data.sync_id", "indices.testing.shards.0.0.commit.user_data") From 2fa23f764d71a1035766cd8efefa5842fe4cb961 Mon Sep 17 00:00:00 2001 From: pgomulka Date: Tue, 27 Jul 2021 17:26:30 +0200 Subject: [PATCH 6/8] duplicate flush_sync using flush and remove unnecesary flush call --- .../upgrades/FullClusterRestartIT.java | 3 -- .../elasticsearch/backwards/IndexingIT.java | 43 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java index b13f5a0303fa4..d6dcd01a3e053 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java @@ -682,9 +682,6 @@ public void testRecovery() throws Exception { flushRequest.addParameter("force", "true"); flushRequest.addParameter("wait_if_ongoing", "true"); assertOK(client().performRequest(flushRequest)); - if (randomBoolean()) { - flush(index, randomBoolean()); - } if (shouldHaveTranslog) { // Update a few documents so we are sure to have a translog diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index 4beea5b1d6e90..daf2b8563b47e 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -332,6 +332,49 @@ public void testSyncedFlushTransition() throws Exception { } } + public void testFlushTransition() throws Exception { + Nodes nodes = buildNodeAndVersions(); + assumeFalse("no new node found", nodes.getNewNodes().isEmpty()); + assumeFalse("no bwc node found", nodes.getBWCNodes().isEmpty()); + // Allocate shards to new nodes then verify synced flush requests processed by old nodes/new nodes + String newNodes = nodes.getNewNodes().stream().map(Node::getNodeName).collect(Collectors.joining(",")); + int numShards = randomIntBetween(1, 10); + int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1); + int totalShards = numShards * (numOfReplicas + 1); + final String index = "test_synced_flush"; + createIndex(index, Settings.builder() + .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas) + .put("index.routing.allocation.include._name", newNodes).build()); + ensureGreen(index); + indexDocs(index, randomIntBetween(0, 100), between(1, 100)); + try (RestClient oldNodeClient = buildClient(restClientSettings(), + nodes.getBWCNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { + Request request = new Request("POST", index + "/_flush"); + assertBusy(() -> { + Map result = ObjectPath.createFromResponse(oldNodeClient.performRequest(request)).evaluate("_shards"); + assertThat(result.get("total"), equalTo(totalShards)); + assertThat(result.get("successful"), equalTo(totalShards)); + assertThat(result.get("failed"), equalTo(0)); + }); + Map stats = entityAsMap(client().performRequest(new Request("GET", index + "/_stats?level=shards"))); + assertThat(XContentMapValues.extractValue("indices." + index + ".total.translog.uncommitted_operations", stats), equalTo(0)); + } + indexDocs(index, randomIntBetween(0, 100), between(1, 100)); + try (RestClient newNodeClient = buildClient(restClientSettings(), + nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { + Request request = new Request("POST", index + "/_flush"); + assertBusy(() -> { + Map result = ObjectPath.createFromResponse(newNodeClient.performRequest(request)).evaluate("_shards"); + assertThat(result.get("total"), equalTo(totalShards)); + assertThat(result.get("successful"), equalTo(totalShards)); + assertThat(result.get("failed"), equalTo(0)); + }); + Map stats = entityAsMap(client().performRequest(new Request("GET", index + "/_stats?level=shards"))); + assertThat(XContentMapValues.extractValue("indices." + index + ".total.translog.uncommitted_operations", stats), equalTo(0)); + } + } + private void assertCount(final String index, final String preference, final int expectedCount) throws IOException { Request request = new Request("GET", index + "/_count"); request.addParameter("preference", preference); From a2a053aeb8c20c4badde693b995d693943740f2c Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Wed, 28 Jul 2021 12:30:48 +0200 Subject: [PATCH 7/8] Update qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java Co-authored-by: David Turner --- .../src/test/java/org/elasticsearch/backwards/IndexingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index daf2b8563b47e..cd0fe70f09235 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -336,7 +336,7 @@ public void testFlushTransition() throws Exception { Nodes nodes = buildNodeAndVersions(); assumeFalse("no new node found", nodes.getNewNodes().isEmpty()); assumeFalse("no bwc node found", nodes.getBWCNodes().isEmpty()); - // Allocate shards to new nodes then verify synced flush requests processed by old nodes/new nodes + // Allocate shards to new nodes then verify flush requests processed by old nodes/new nodes String newNodes = nodes.getNewNodes().stream().map(Node::getNodeName).collect(Collectors.joining(",")); int numShards = randomIntBetween(1, 10); int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1); From bf35bc4d1f23e66e1d50488757484f5b77ceaa46 Mon Sep 17 00:00:00 2001 From: Przemyslaw Gomulka Date: Wed, 28 Jul 2021 12:32:00 +0200 Subject: [PATCH 8/8] Update qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java Co-authored-by: David Turner --- .../src/test/java/org/elasticsearch/backwards/IndexingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java index cd0fe70f09235..e60c8a4f45038 100644 --- a/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java @@ -341,7 +341,7 @@ public void testFlushTransition() throws Exception { int numShards = randomIntBetween(1, 10); int numOfReplicas = randomIntBetween(0, nodes.getNewNodes().size() - 1); int totalShards = numShards * (numOfReplicas + 1); - final String index = "test_synced_flush"; + final String index = "test_flush"; createIndex(index, Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), numShards) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)