Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split the remote global metadata file to metadata attribute files #12190

Merged
merged 29 commits into from
May 16, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
6bf7bc9
Split the cluster state remote global metadata file to metadata attri…
shiv0408 Jan 24, 2024
5d6a0ad
Upload all the metadata attributes in parallel
shiv0408 Feb 8, 2024
279dbbe
Added javadocs
shiv0408 Feb 20, 2024
0542849
Address PR comment
shiv0408 Feb 29, 2024
c9fb25b
Add Integ test
shiv0408 Mar 12, 2024
fc270d1
Added a bwc test
shiv0408 Mar 13, 2024
adb4cf2
Address PR comments
shiv0408 Mar 14, 2024
0b38736
Modify Custom's fromXContent
shiv0408 Mar 15, 2024
c86c0f1
Remove ClusterMetadataManifest constructor
shiv0408 Mar 18, 2024
3ed92e5
Added tests
shiv0408 Mar 18, 2024
e1f517e
remove stale global metadata files
shiv0408 Mar 19, 2024
cd5c9a5
spotless apply
shiv0408 Mar 19, 2024
2bd97d7
Merge branch 'main' into cluster_state_split
shiv0408 Mar 19, 2024
8244c6d
Merge branch 'main' into cluster_state_split
shiv0408 Apr 8, 2024
425cf20
Merge branch 'main' into cluster_state_split
shiv0408 Apr 29, 2024
8efc1ed
fix build after merge
shiv0408 Apr 29, 2024
6c637c0
Merge branch 'main' into cluster_state_split
shiv0408 Apr 30, 2024
494aacc
Address PR comments
shiv0408 Apr 30, 2024
928b650
Address further PR comment
shiv0408 May 2, 2024
2ebfc6d
apply spotless
shiv0408 May 2, 2024
fb0b6aa
removed unnecessary method
shiv0408 May 3, 2024
032ced2
Address further PR comments
shiv0408 May 10, 2024
dd76f39
Merge branch 'main' into cluster_state_split
shiv0408 May 10, 2024
e81db21
Fix test failures
shiv0408 May 13, 2024
b20537a
Merge branch 'main' into cluster_state_split
shiv0408 May 13, 2024
87893ad
Change TemplatesMetadata to API
shiv0408 May 14, 2024
2fe0dd6
updated API version for TemplatesMetadata.Builder
shiv0408 May 15, 2024
556278f
Merge branch 'main' into cluster_state_split
shiv0408 May 15, 2024
4f8a64e
Merge branch 'main' into cluster_state_split
shiv0408 May 15, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -752,6 +752,10 @@ public String getIndexUUID() {
return index.getUUID();
}

public String getIndexName() {
return index.getName();
}

shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
/**
* Test whether the current index UUID is the same as the given one. Returns true if either are _na_
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,27 @@ public enum XContentContext {
public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, ClusterState.FeatureAware {

EnumSet<XContentContext> context();

static Custom fromXContent(XContentParser parser, String name) throws IOException {
try {
return parser.namedObject(Custom.class, name, null);
} catch (NamedObjectNotFoundException e) {
logger.warn("Unknown custom object with type {}", name);
parser.skipChildren();
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
throw e;
}
}

static Custom fromXContent(XContentParser parser) throws IOException {
String currentFieldName = parser.currentName();
try {
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
return parser.namedObject(Custom.class, currentFieldName, null);
} catch (NamedObjectNotFoundException e) {
logger.warn("Unknown custom object with type {}", currentFieldName);
parser.skipChildren();
throw e;
}
}
}

public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
Expand Down Expand Up @@ -260,7 +281,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
private final Settings settings;
private final DiffableStringMap hashesOfConsistentSettings;
private final Map<String, IndexMetadata> indices;
private final Map<String, IndexTemplateMetadata> templates;
private final TemplatesMetadata templates;
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
private final Map<String, Custom> customs;

private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
Expand Down Expand Up @@ -304,7 +325,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
this.hashesOfConsistentSettings = hashesOfConsistentSettings;
this.indices = Collections.unmodifiableMap(indices);
this.customs = Collections.unmodifiableMap(customs);
this.templates = Collections.unmodifiableMap(templates);
this.templates = new TemplatesMetadata(templates);
int totalNumberOfShards = 0;
int totalOpenIndexShards = 0;
for (IndexMetadata cursor : indices.values()) {
Expand Down Expand Up @@ -806,13 +827,17 @@ public Map<String, IndexMetadata> getIndices() {
}

public Map<String, IndexTemplateMetadata> templates() {
return this.templates;
return this.templates.getTemplates();
}

public Map<String, IndexTemplateMetadata> getTemplates() {
return templates();
}

public TemplatesMetadata templatesMetadata() {
return this.templates;
}

public Map<String, ComponentTemplate> componentTemplates() {
return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
.map(ComponentTemplateMetadata::componentTemplates)
Expand Down Expand Up @@ -923,7 +948,7 @@ public Iterator<IndexMetadata> iterator() {
}

public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
return false;
}
if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
Expand All @@ -942,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
* Compares Metadata entities persisted in Remote Store.
*/
public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
if (!isSettingsMetadataEqual(metadata1, metadata2)) {
return false;
}
if (!metadata1.templates.equals(metadata2.templates())) {
if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
return false;
}
// Check if any persistent metadata needs to be saved
return isCustomMetadataEqual(metadata1, metadata2);
}

public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
}

public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
return metadata1.persistentSettings.equals(metadata2.persistentSettings);
}

public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
return metadata1.templates.equals(metadata2.templates);
}

public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
int customCount1 = 0;
for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
Expand All @@ -962,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
customCount2++;
}
}
if (customCount1 != customCount2) return false;
return true;
return customCount1 == customCount2;
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
Expand Down Expand Up @@ -1012,7 +1052,11 @@ private static class MetadataDiff implements Diff<Metadata> {
persistentSettings = after.persistentSettings;
hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
templates = DiffableUtils.diff(
before.templates.getTemplates(),
after.templates.getTemplates(),
DiffableUtils.getStringKeySerializer()
);
customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
}

Expand Down Expand Up @@ -1059,7 +1103,7 @@ public Metadata apply(Metadata part) {
builder.persistentSettings(persistentSettings);
builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
builder.indices(indices.apply(part.indices));
builder.templates(templates.apply(part.templates));
builder.templates(templates.apply(part.templates.getTemplates()));
builder.customs(customs.apply(part.customs));
return builder.build();
}
Expand Down Expand Up @@ -1103,10 +1147,7 @@ public void writeTo(StreamOutput out) throws IOException {
for (IndexMetadata indexMetadata : this) {
indexMetadata.writeTo(out);
}
out.writeVInt(templates.size());
for (final IndexTemplateMetadata cursor : templates.values()) {
cursor.writeTo(out);
}
templates.writeTo(out);
// filter out custom states not supported by the other node
int numberOfCustoms = 0;
for (final Custom cursor : customs.values()) {
Expand Down Expand Up @@ -1170,7 +1211,7 @@ public Builder(Metadata metadata) {
this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
this.version = metadata.version;
this.indices = new HashMap<>(metadata.indices);
this.templates = new HashMap<>(metadata.templates);
this.templates = new HashMap<>(metadata.templates.getTemplates());
this.customs = new HashMap<>(metadata.customs);
this.previousMetadata = metadata;
}
Expand Down Expand Up @@ -1249,6 +1290,11 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
return this;
}

public Builder templates(TemplatesMetadata templatesMetadata) {
this.templates.putAll(templatesMetadata.getTemplates());
return this;
}

public Builder put(String name, ComponentTemplate componentTemplate) {
Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
Map<String, ComponentTemplate> existingTemplates = Optional.ofNullable(
Expand Down Expand Up @@ -1739,9 +1785,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
}

builder.startObject("templates");
for (final IndexTemplateMetadata cursor : metadata.templates().values()) {
IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
}
metadata.templatesMetadata().toXContent(builder, params);
builder.endObject();

if (context == XContentContext.API) {
Expand Down Expand Up @@ -1804,16 +1848,13 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
} else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
builder.hashesOfConsistentSettings(parser.mapStrings());
} else if ("templates".equals(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
}
builder.templates(TemplatesMetadata.fromXContent(parser));
} else {
try {
Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
Custom custom = Custom.fromXContent(parser, currentFieldName);
builder.putCustom(custom.getWriteableName(), custom);
} catch (NamedObjectNotFoundException ex) {
logger.warn("Skipping unknown custom object with type {}", currentFieldName);
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
parser.skipChildren();
}
}
} else if (token.isValue()) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
XContentParser.Token token;
List<RepositoryMetadata> repository = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.START_OBJECT) {
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
// move to next token if parsing the whole object
token = parser.nextToken();
}
if (token == XContentParser.Token.FIELD_NAME) {
String name = parser.currentName();
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.cluster.metadata;

import org.opensearch.cluster.AbstractDiffable;
import org.opensearch.core.common.io.stream.StreamOutput;
import org.opensearch.core.xcontent.ToXContentFragment;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.core.xcontent.XContentParser;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

public class TemplatesMetadata extends AbstractDiffable<TemplatesMetadata> implements ToXContentFragment {
public static TemplatesMetadata EMPTY_METADATA = builder().build();
private final Map<String, IndexTemplateMetadata> templates;

public TemplatesMetadata() {
this(Collections.emptyMap());
}

public TemplatesMetadata(Map<String, IndexTemplateMetadata> templates) {
this.templates = Collections.unmodifiableMap(templates);
}

public static Builder builder() {
return new Builder();
}

public Map<String, IndexTemplateMetadata> getTemplates() {
return this.templates;
}

public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
return Builder.fromXContent(parser);
}

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
Builder.toXContent(this, builder, params);
return builder;
}

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(templates.size());
for (final IndexTemplateMetadata cursor : templates.values()) {
cursor.writeTo(out);
}
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;

TemplatesMetadata that = (TemplatesMetadata) o;

return Objects.equals(templates, that.templates);
}

@Override
public int hashCode() {
return templates != null ? templates.hashCode() : 0;
}

public static class Builder {
private final Map<String, IndexTemplateMetadata> templates;

public Builder() {
this.templates = new HashMap<String, IndexTemplateMetadata>();
shwetathareja marked this conversation as resolved.
Show resolved Hide resolved
}

public Builder(Map<String, IndexTemplateMetadata> templates) {
this.templates = templates;
}

public Builder put(IndexTemplateMetadata.Builder template) {
return put(template.build());
}

public Builder put(IndexTemplateMetadata template) {
templates.put(template.name(), template);
return this;
}

public Builder removeTemplate(String templateName) {
templates.remove(templateName);
return this;
}

public Builder templates(Map<String, IndexTemplateMetadata> templates) {
this.templates.putAll(templates);
return this;
}

public TemplatesMetadata build() {
return new TemplatesMetadata(templates);
}

public static void toXContent(TemplatesMetadata templates, XContentBuilder builder, Params params) throws IOException {
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
// builder.startObject("templates-metadata");
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
for (IndexTemplateMetadata cursor : templates.getTemplates().values()) {
IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
}
// builder.endObject();
}

public static TemplatesMetadata fromXContent(XContentParser parser) throws IOException {
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
Builder builder = new Builder();

// we might get here after the templates-metadata element, or on a fresh parser
shiv0408 marked this conversation as resolved.
Show resolved Hide resolved
XContentParser.Token token = parser.currentToken();
String currentFieldName = parser.currentName();
if (currentFieldName == null) {
token = parser.nextToken();
if (token == XContentParser.Token.START_OBJECT) {
// move to the field name
token = parser.nextToken();
if (token == XContentParser.Token.FIELD_NAME) {
// move to the next object
token = parser.nextToken();
}
}
currentFieldName = parser.currentName();
}
if (currentFieldName != null && token == XContentParser.Token.START_OBJECT) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
}
}
return builder.build();
}
}
}
Loading
Loading