From 8da98e20cb9379c27ea793e98f1dc315affcee3d Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 30 Jun 2020 12:35:08 -0400 Subject: [PATCH] Fix testFailOverOnFollower --- .../test/InternalTestCluster.java | 9 +++++++++ .../xpack/ccr/FollowerFailOverIT.java | 7 +++---- .../elasticsearch/xpack/CcrIntegTestCase.java | 19 +++++++++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 1fbfd11603ace..60975270bebd8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2087,6 +2087,15 @@ private Collection dataNodeAndClients() { return filterNodes(nodes, DATA_NODE_PREDICATE); } + public Collection filterNodes(Predicate predicate) { + return nodes + .values() + .stream() + .map(NodeAndClient::node) + .filter(predicate) + .collect(Collectors.toCollection(ArrayList::new)); + } + private static Collection filterNodes(Map map, Predicate predicate) { return map diff --git a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java index 6bc6e0e24e9aa..0e48b531a1979 100644 --- a/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java +++ b/x-pack/plugin/ccr/src/internalClusterTest/java/org/elasticsearch/xpack/ccr/FollowerFailOverIT.java @@ -51,13 +51,12 @@ protected boolean reuseClusters() { return false; } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/58534") public void testFailOverOnFollower() throws Exception { final String leaderIndex = "leader_test_failover"; final String followerIndex = "follower_test_failover"; int numberOfReplicas = between(1, 2); getFollowerCluster().startMasterOnlyNode(); - getFollowerCluster().ensureAtLeastNumDataNodes(numberOfReplicas + between(1, 2)); + ensureFollowerHasAtLeastNumDataAndRemoteClusterClientNodes(numberOfReplicas + between(1, 2)); String leaderIndexSettings = getIndexSettings(1, numberOfReplicas); assertAcked(leaderClient().admin().indices().prepareCreate(leaderIndex).setSource(leaderIndexSettings, XContentType.JSON)); AtomicBoolean stopped = new AtomicBoolean(); @@ -122,7 +121,7 @@ public void testFailOverOnFollower() throws Exception { } public void testFollowIndexAndCloseNode() throws Exception { - getFollowerCluster().ensureAtLeastNumDataNodes(3); + ensureFollowerHasAtLeastNumDataAndRemoteClusterClientNodes(3); String leaderIndexSettings = getIndexSettings(3, 1); assertAcked(leaderClient().admin().indices().prepareCreate("index1").setSource(leaderIndexSettings, XContentType.JSON)); ensureLeaderGreen("index1"); @@ -181,7 +180,7 @@ public void testAddNewReplicasOnFollower() throws Exception { assertAcked(leaderClient().admin().indices().prepareCreate("leader-index").setSource(leaderIndexSettings, XContentType.JSON)); PutFollowAction.Request follow = putFollow("leader-index", "follower-index"); followerClient().execute(PutFollowAction.INSTANCE, follow).get(); - getFollowerCluster().ensureAtLeastNumDataNodes(numberOfReplicas + between(2, 3)); + ensureFollowerHasAtLeastNumDataAndRemoteClusterClientNodes(numberOfReplicas + between(2, 3)); ensureFollowerGreen("follower-index"); AtomicBoolean stopped = new AtomicBoolean(); AtomicInteger docID = new AtomicInteger(); diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java index 88bbeccc7de0e..f5f410445e38c 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/CcrIntegTestCase.java @@ -31,6 +31,8 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; @@ -73,6 +75,7 @@ import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.test.NodeConfigurationSource; +import org.elasticsearch.test.NodeRoles; import org.elasticsearch.test.TestCluster; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.transport.RemoteConnectionStrategy; @@ -751,6 +754,22 @@ public void clusterStateProcessed(String source, ClusterState oldState, ClusterS latch.await(); } + protected void ensureFollowerHasAtLeastNumDataAndRemoteClusterClientNodes(int n) { + final InternalTestCluster followerCluster = getFollowerCluster(); + final int currentNodes = followerCluster.filterNodes(node -> + DiscoveryNode.hasRole(node.settings(), DiscoveryNodeRole.DATA_ROLE) && + DiscoveryNode.hasRole(node.settings(), DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)).size(); + if (currentNodes < n) { + followerCluster.startNodes(n - currentNodes, + NodeRoles.onlyRoles(Settings.EMPTY, Set.of(DiscoveryNodeRole.DATA_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE))); + followerCluster.validateClusterFormed(); + } + final int newNodes = followerCluster.filterNodes(node -> + DiscoveryNode.hasRole(node.settings(), DiscoveryNodeRole.DATA_ROLE) && + DiscoveryNode.hasRole(node.settings(), DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE)).size(); + assertThat(newNodes, greaterThanOrEqualTo(n)); + } + static class ClusterGroup implements Closeable { final InternalTestCluster leaderCluster;