Skip to content

Commit

Permalink
Fix cluster chaining during bootstrap (opensearch-project#10020)
Browse files Browse the repository at this point in the history 
* Fix clusterUUID chaining logic

Signed-off-by: Sooraj Sinha <soosinha@amazon.com>
Signed-off-by: Shivansh Arora <hishiv@amazon.com>
  • Loading branch information
@soosinha @shiv0408
soosinha authored and shiv0408 committed Apr 25, 2024
1 parent d1e6569 commit dff7b29
Show file tree
Hide file tree
Showing 7 changed files with 315 additions and 60 deletions.
43 changes: 37 additions & 6 deletions server/src/main/java/org/opensearch/gateway/GatewayMetaState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
Expand Down Expand Up @@ -171,11 +172,12 @@ public void start(
// If the cluster UUID loaded from local is unknown (_na_) then fetch the best state from remote
// If there is no valid state on remote, continue with initial empty state
// If there is a valid state, then restore index metadata using this state
String lastKnownClusterUUID = ClusterState.UNKNOWN_UUID;
if (ClusterState.UNKNOWN_UUID.equals(clusterState.metadata().clusterUUID())) {
String lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
lastKnownClusterUUID = remoteClusterStateService.getLastKnownUUIDFromRemote(
clusterState.getClusterName().value()
);
if (!ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID)) {
if (ClusterState.UNKNOWN_UUID.equals(lastKnownClusterUUID) == false) {
// Load state from remote
final RemoteRestoreResult remoteRestoreResult = remoteStoreRestoreService.restore(
clusterState,
Expand All @@ -186,7 +188,7 @@ public void start(
clusterState = remoteRestoreResult.getClusterState();
}
}
remotePersistedState = new RemotePersistedState(remoteClusterStateService);
remotePersistedState = new RemotePersistedState(remoteClusterStateService, lastKnownClusterUUID);
}
persistedState = new LucenePersistedState(persistedClusterStateService, currentTerm, clusterState);
} else {
Expand Down Expand Up @@ -647,9 +649,11 @@ public static class RemotePersistedState implements PersistedState {
private ClusterState lastAcceptedState;
private ClusterMetadataManifest lastAcceptedManifest;
private final RemoteClusterStateService remoteClusterStateService;
private String previousClusterUUID;

public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService) {
public RemotePersistedState(final RemoteClusterStateService remoteClusterStateService, final String previousClusterUUID) {
this.remoteClusterStateService = remoteClusterStateService;
this.previousClusterUUID = previousClusterUUID;
}

@Override
Expand All @@ -674,7 +678,26 @@ public void setLastAcceptedState(ClusterState clusterState) {
try {
final ClusterMetadataManifest manifest;
if (shouldWriteFullClusterState(clusterState)) {
manifest = remoteClusterStateService.writeFullMetadata(clusterState);
if (clusterState.metadata().clusterUUIDCommitted() == true) {
final Optional<ClusterMetadataManifest> latestManifest = remoteClusterStateService.getLatestClusterMetadataManifest(
clusterState.getClusterName().value(),
clusterState.metadata().clusterUUID()
);
if (latestManifest.isPresent()) {
// The previous UUID should not change for the current UUID. So fetching the latest manifest
// from remote store and getting the previous UUID.
previousClusterUUID = latestManifest.get().getPreviousClusterUUID();
} else {
// When the user starts the cluster with remote state disabled but later enables the remote state,
// there will not be any manifest for the current cluster UUID.
logger.error(
"Latest manifest is not present in remote store for cluster UUID: {}",
clusterState.metadata().clusterUUID()
);
previousClusterUUID = ClusterState.UNKNOWN_UUID;
}
}
manifest = remoteClusterStateService.writeFullMetadata(clusterState, previousClusterUUID);
} else {
assert verifyManifestAndClusterState(lastAcceptedManifest, lastAcceptedState) == true
: "Previous manifest and previous ClusterState are not in sync";
Expand Down Expand Up @@ -723,11 +746,19 @@ public void markLastAcceptedStateAsCommitted() {
try {
assert lastAcceptedState != null : "Last accepted state is not present";
assert lastAcceptedManifest != null : "Last accepted manifest is not present";
ClusterState clusterState = lastAcceptedState;
if (lastAcceptedState.metadata().clusterUUID().equals(Metadata.UNKNOWN_CLUSTER_UUID) == false
&& lastAcceptedState.metadata().clusterUUIDCommitted() == false) {
Metadata.Builder metadataBuilder = Metadata.builder(lastAcceptedState.metadata());
metadataBuilder.clusterUUIDCommitted(true);
clusterState = ClusterState.builder(lastAcceptedState).metadata(metadataBuilder).build();
}
final ClusterMetadataManifest committedManifest = remoteClusterStateService.markLastStateAsCommitted(
lastAcceptedState,
clusterState,
lastAcceptedManifest
);
lastAcceptedManifest = committedManifest;
lastAcceptedState = clusterState;
} catch (Exception e) {
handleExceptionOnWrite(e);
}
Expand Down
37 changes: 32 additions & 5 deletions server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ public class ClusterMetadataManifest implements Writeable, ToXContentFragment {
private static final ParseField COMMITTED_FIELD = new ParseField("committed");
private static final ParseField INDICES_FIELD = new ParseField("indices");
private static final ParseField PREVIOUS_CLUSTER_UUID = new ParseField("previous_cluster_uuid");
private static final ParseField CLUSTER_UUID_COMMITTED = new ParseField("cluster_uuid_committed");

private static long term(Object[] fields) {
return (long) fields[0];
Expand Down Expand Up @@ -79,6 +80,10 @@ private static String previousClusterUUID(Object[] fields) {
return (String) fields[8];
}

private static boolean clusterUUIDCommitted(Object[] fields) {
return (boolean) fields[9];
}

private static final ConstructingObjectParser<ClusterMetadataManifest, Void> PARSER = new ConstructingObjectParser<>(
"cluster_metadata_manifest",
fields -> new ClusterMetadataManifest(
Expand All @@ -90,7 +95,8 @@ private static String previousClusterUUID(Object[] fields) {
nodeId(fields),
committed(fields),
indices(fields),
previousClusterUUID(fields)
previousClusterUUID(fields),
clusterUUIDCommitted(fields)
)
);

Expand All @@ -108,6 +114,7 @@ private static String previousClusterUUID(Object[] fields) {
INDICES_FIELD
);
PARSER.declareString(ConstructingObjectParser.constructorArg(), PREVIOUS_CLUSTER_UUID);
PARSER.declareBoolean(ConstructingObjectParser.constructorArg(), CLUSTER_UUID_COMMITTED);
}

private final List<UploadedIndexMetadata> indices;
Expand All @@ -119,6 +126,7 @@ private static String previousClusterUUID(Object[] fields) {
private final String nodeId;
private final boolean committed;
private final String previousClusterUUID;
private final boolean clusterUUIDCommitted;

public List<UploadedIndexMetadata> getIndices() {
return indices;
Expand Down Expand Up @@ -156,6 +164,10 @@ public String getPreviousClusterUUID() {
return previousClusterUUID;
}

public boolean isClusterUUIDCommitted() {
return clusterUUIDCommitted;
}

public ClusterMetadataManifest(
long clusterTerm,
long version,
Expand All @@ -165,7 +177,8 @@ public ClusterMetadataManifest(
String nodeId,
boolean committed,
List<UploadedIndexMetadata> indices,
String previousClusterUUID
String previousClusterUUID,
boolean clusterUUIDCommitted
) {
this.clusterTerm = clusterTerm;
this.stateVersion = version;
Expand All @@ -176,6 +189,7 @@ public ClusterMetadataManifest(
this.committed = committed;
this.indices = Collections.unmodifiableList(indices);
this.previousClusterUUID = previousClusterUUID;
this.clusterUUIDCommitted = clusterUUIDCommitted;
}

public ClusterMetadataManifest(StreamInput in) throws IOException {
Expand All @@ -188,6 +202,7 @@ public ClusterMetadataManifest(StreamInput in) throws IOException {
this.committed = in.readBoolean();
this.indices = Collections.unmodifiableList(in.readList(UploadedIndexMetadata::new));
this.previousClusterUUID = in.readString();
this.clusterUUIDCommitted = in.readBoolean();
}

public static Builder builder() {
Expand Down Expand Up @@ -215,6 +230,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();
builder.field(PREVIOUS_CLUSTER_UUID.getPreferredName(), getPreviousClusterUUID());
builder.field(CLUSTER_UUID_COMMITTED.getPreferredName(), isClusterUUIDCommitted());
return builder;
}

Expand All @@ -229,6 +245,7 @@ public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(committed);
out.writeCollection(indices);
out.writeString(previousClusterUUID);
out.writeBoolean(clusterUUIDCommitted);
}

@Override
Expand All @@ -248,7 +265,8 @@ public boolean equals(Object o) {
&& Objects.equals(opensearchVersion, that.opensearchVersion)
&& Objects.equals(nodeId, that.nodeId)
&& Objects.equals(committed, that.committed)
&& Objects.equals(previousClusterUUID, that.previousClusterUUID);
&& Objects.equals(previousClusterUUID, that.previousClusterUUID)
&& Objects.equals(clusterUUIDCommitted, that.clusterUUIDCommitted);
}

@Override
Expand All @@ -262,7 +280,8 @@ public int hashCode() {
opensearchVersion,
nodeId,
committed,
previousClusterUUID
previousClusterUUID,
clusterUUIDCommitted
);
}

Expand Down Expand Up @@ -291,6 +310,7 @@ public static class Builder {
private String nodeId;
private String previousClusterUUID;
private boolean committed;
private boolean clusterUUIDCommitted;

public Builder indices(List<UploadedIndexMetadata> indices) {
this.indices = indices;
Expand Down Expand Up @@ -341,6 +361,11 @@ public Builder previousClusterUUID(String previousClusterUUID) {
return this;
}

public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) {
this.clusterUUIDCommitted = clusterUUIDCommitted;
return this;
}

public Builder() {
indices = new ArrayList<>();
}
Expand All @@ -355,6 +380,7 @@ public Builder(ClusterMetadataManifest manifest) {
this.committed = manifest.committed;
this.indices = new ArrayList<>(manifest.indices);
this.previousClusterUUID = manifest.previousClusterUUID;
this.clusterUUIDCommitted = manifest.clusterUUIDCommitted;
}

public ClusterMetadataManifest build() {
Expand All @@ -367,7 +393,8 @@ public ClusterMetadataManifest build() {
nodeId,
committed,
indices,
previousClusterUUID
previousClusterUUID,
clusterUUIDCommitted
);
}

Expand Down
Loading

0 comments on commit dff7b29

Please sign in to comment.