Split the remote global metadata file to metadata attribute files

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/ClusterStateIT.java

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.upgrades;
+
+import org.opensearch.client.Request;
+import org.opensearch.client.Response;
+
+import java.util.Map;
+
+public class ClusterStateIT extends AbstractRollingTestCase{
+    public void testTemplateMetadataUpgrades() throws Exception {
+        if (CLUSTER_TYPE == ClusterType.OLD) {
+            String templateName = "my_template";
+            Request putIndexTemplate = new Request("PUT", "_template/" + templateName);
+            putIndexTemplate.setJsonEntity("{\"index_patterns\": [\"pattern-1\", \"log-*\"]}");
+            client().performRequest(putIndexTemplate);
+            verifyTemplateMetadataInClusterState();
+        } else {
+            verifyTemplateMetadataInClusterState();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void verifyTemplateMetadataInClusterState() throws Exception {
+        Request request = new Request("GET", "_cluster/state/metadata");
+        Response response = client().performRequest(request);
+        assertOK(response);
+        Map<String, Object> metadata = (Map<String, Object>) entityAsMap(response).get("metadata");
+        assertNotNull(metadata.get("templates"));
+    }
+}

server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteClusterStateServiceIT.java

@@ -19,13 +19,21 @@
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
+import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.COORDINATION_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.CUSTOM_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.DELIMITER;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.METADATA_FILE_PREFIX;
 import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.SETTING_METADATA;
+import static org.opensearch.gateway.remote.RemoteClusterStateService.TEMPLATES_METADATA;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteClusterStateServiceIT extends RemoteStoreBaseIntegTestCase {
@@ -181,6 +189,45 @@ public void testRemoteStateStatsFromAllNodes() {
         }
     }
 
+    public void testRemoteClusterStateMetadataSplit() throws IOException {
+        initialTestSetup(1, 0, 1, 1);
+
+        RemoteClusterStateService remoteClusterStateService = internalCluster().getClusterManagerNodeInstance(
+            RemoteClusterStateService.class
+        );
+        RepositoriesService repositoriesService = internalCluster().getClusterManagerNodeInstance(RepositoriesService.class);
+        BlobStoreRepository repository = (BlobStoreRepository) repositoriesService.repository(REPOSITORY_NAME);
+        BlobPath globalMetadataPath = repository.basePath()
+            .add(
+                Base64.getUrlEncoder()
+                    .withoutPadding()
+                    .encodeToString(getClusterState().getClusterName().value().getBytes(StandardCharsets.UTF_8))
+            )
+            .add("cluster-state")
+            .add(getClusterState().metadata().clusterUUID())
+            .add("global-metadata");
+
+        Map<String, Integer> metadataFiles = repository.blobStore()
+            .blobContainer(globalMetadataPath)
+            .listBlobs()
+            .keySet()
+            .stream()
+            .map(fileName -> {
+                logger.info(fileName);
+                return fileName.split(DELIMITER)[0];
+            })
+            .collect(Collectors.toMap(Function.identity(), key -> 1, Integer::sum));
+
+        assertTrue(metadataFiles.containsKey(COORDINATION_METADATA));
+        assertEquals(1, (int) metadataFiles.get(COORDINATION_METADATA));
+        assertTrue(metadataFiles.containsKey(SETTING_METADATA));
+        assertEquals(1, (int) metadataFiles.get(SETTING_METADATA));
+        assertTrue(metadataFiles.containsKey(TEMPLATES_METADATA));
+        assertEquals(1, (int) metadataFiles.get(TEMPLATES_METADATA));
+        assertTrue(metadataFiles.keySet().stream().anyMatch(key -> key.startsWith(CUSTOM_METADATA)));
+        assertFalse(metadataFiles.containsKey(METADATA_FILE_PREFIX));
+    }
+
     private void validateNodesStatsResponse(NodesStatsResponse nodesStatsResponse) {
         // _nodes/stats/discovery must never fail due to any exception
         assertFalse(nodesStatsResponse.toString().contains("exception"));

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -752,6 +752,10 @@ public String getIndexUUID() {
         return index.getUUID();
     }
 
+    public String getIndexName() {
+        return index.getName();
+    }
+
     /**
      * Test whether the current index UUID is the same as the given one. Returns true if either are _na_
      */

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -175,6 +175,16 @@ public enum XContentContext {
     public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, ClusterState.FeatureAware {
 
         EnumSet<XContentContext> context();
+
+        static Custom fromXContent(XContentParser parser, String name) throws IOException {
+            // handling any Exception is caller's responsibility
+            return parser.namedObject(Custom.class, name, null);
+        }
+
+        static Custom fromXContent(XContentParser parser) throws IOException {
+            String currentFieldName = parser.currentName();
+            return fromXContent(parser, currentFieldName);
+        }
     }
 
     public static final Setting<Integer> DEFAULT_REPLICA_COUNT_SETTING = Setting.intSetting(
@@ -260,7 +270,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
     private final Settings settings;
     private final DiffableStringMap hashesOfConsistentSettings;
     private final Map<String, IndexMetadata> indices;
-    private final Map<String, IndexTemplateMetadata> templates;
+    private final TemplatesMetadata templates;
     private final Map<String, Custom> customs;
 
     private final transient int totalNumberOfShards; // Transient ? not serializable anyway?
@@ -304,7 +314,7 @@ public interface Custom extends NamedDiffable<Custom>, ToXContentFragment, Clust
         this.hashesOfConsistentSettings = hashesOfConsistentSettings;
         this.indices = Collections.unmodifiableMap(indices);
         this.customs = Collections.unmodifiableMap(customs);
-        this.templates = Collections.unmodifiableMap(templates);
+        this.templates = new TemplatesMetadata(templates);
         int totalNumberOfShards = 0;
         int totalOpenIndexShards = 0;
         for (IndexMetadata cursor : indices.values()) {
@@ -806,13 +816,17 @@ public Map<String, IndexMetadata> getIndices() {
     }
 
     public Map<String, IndexTemplateMetadata> templates() {
-        return this.templates;
+        return this.templates.getTemplates();
     }
 
     public Map<String, IndexTemplateMetadata> getTemplates() {
         return templates();
     }
 
+    public TemplatesMetadata templatesMetadata() {
+        return this.templates;
+    }
+
     public Map<String, ComponentTemplate> componentTemplates() {
         return Optional.ofNullable((ComponentTemplateMetadata) this.custom(ComponentTemplateMetadata.TYPE))
             .map(ComponentTemplateMetadata::componentTemplates)
@@ -923,7 +937,7 @@ public Iterator<IndexMetadata> iterator() {
     }
 
     public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
+        if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
@@ -942,13 +956,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
      * Compares Metadata entities persisted in Remote Store.
      */
     public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!isSettingsMetadataEqual(metadata1, metadata2)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates())) {
+        if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
+        return isCustomMetadataEqual(metadata1, metadata2);
+    }
+
+    public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
+    }
+
+    public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.persistentSettings.equals(metadata2.persistentSettings);
+    }
+
+    public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.templates.equals(metadata2.templates);
+    }
+
+    public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
         int customCount1 = 0;
         for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
             if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
@@ -962,8 +992,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
                 customCount2++;
             }
         }
-        if (customCount1 != customCount2) return false;
-        return true;
+        return customCount1 == customCount2;
     }
 
     @Override
@@ -1012,7 +1041,11 @@ private static class MetadataDiff implements Diff<Metadata> {
             persistentSettings = after.persistentSettings;
             hashesOfConsistentSettings = after.hashesOfConsistentSettings.diff(before.hashesOfConsistentSettings);
             indices = DiffableUtils.diff(before.indices, after.indices, DiffableUtils.getStringKeySerializer());
-            templates = DiffableUtils.diff(before.templates, after.templates, DiffableUtils.getStringKeySerializer());
+            templates = DiffableUtils.diff(
+                before.templates.getTemplates(),
+                after.templates.getTemplates(),
+                DiffableUtils.getStringKeySerializer()
+            );
             customs = DiffableUtils.diff(before.customs, after.customs, DiffableUtils.getStringKeySerializer(), CUSTOM_VALUE_SERIALIZER);
         }
 
@@ -1059,7 +1092,7 @@ public Metadata apply(Metadata part) {
             builder.persistentSettings(persistentSettings);
             builder.hashesOfConsistentSettings(hashesOfConsistentSettings.apply(part.hashesOfConsistentSettings));
             builder.indices(indices.apply(part.indices));
-            builder.templates(templates.apply(part.templates));
+            builder.templates(templates.apply(part.templates.getTemplates()));
             builder.customs(customs.apply(part.customs));
             return builder.build();
         }
@@ -1103,10 +1136,7 @@ public void writeTo(StreamOutput out) throws IOException {
         for (IndexMetadata indexMetadata : this) {
             indexMetadata.writeTo(out);
         }
-        out.writeVInt(templates.size());
-        for (final IndexTemplateMetadata cursor : templates.values()) {
-            cursor.writeTo(out);
-        }
+        templates.writeTo(out);
         // filter out custom states not supported by the other node
         int numberOfCustoms = 0;
         for (final Custom cursor : customs.values()) {
@@ -1170,7 +1200,7 @@ public Builder(Metadata metadata) {
             this.hashesOfConsistentSettings = metadata.hashesOfConsistentSettings;
             this.version = metadata.version;
             this.indices = new HashMap<>(metadata.indices);
-            this.templates = new HashMap<>(metadata.templates);
+            this.templates = new HashMap<>(metadata.templates.getTemplates());
             this.customs = new HashMap<>(metadata.customs);
             this.previousMetadata = metadata;
         }
@@ -1249,6 +1279,11 @@ public Builder templates(Map<String, IndexTemplateMetadata> templates) {
             return this;
         }
 
+        public Builder templates(TemplatesMetadata templatesMetadata) {
+            this.templates.putAll(templatesMetadata.getTemplates());
+            return this;
+        }
+
         public Builder put(String name, ComponentTemplate componentTemplate) {
             Objects.requireNonNull(componentTemplate, "it is invalid to add a null component template: " + name);
             Map<String, ComponentTemplate> existingTemplates = Optional.ofNullable(
@@ -1739,9 +1774,7 @@ public static void toXContent(Metadata metadata, XContentBuilder builder, ToXCon
             }
 
             builder.startObject("templates");
-            for (final IndexTemplateMetadata cursor : metadata.templates().values()) {
-                IndexTemplateMetadata.Builder.toXContentWithTypes(cursor, builder, params);
-            }
+            metadata.templatesMetadata().toXContent(builder, params);
             builder.endObject();
 
             if (context == XContentContext.API) {
@@ -1804,12 +1837,10 @@ public static Metadata fromXContent(XContentParser parser) throws IOException {
                     } else if ("hashes_of_consistent_settings".equals(currentFieldName)) {
                         builder.hashesOfConsistentSettings(parser.mapStrings());
                     } else if ("templates".equals(currentFieldName)) {
-                        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
-                            builder.put(IndexTemplateMetadata.Builder.fromXContent(parser, parser.currentName()));
-                        }
+                        builder.templates(TemplatesMetadata.fromXContent(parser));
                     } else {
                         try {
-                            Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
+                            Custom custom = Custom.fromXContent(parser, currentFieldName);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
                             logger.warn("Skipping unknown custom object with type {}", currentFieldName);

server/src/main/java/org/opensearch/cluster/metadata/RepositoriesMetadata.java

@@ -202,6 +202,10 @@ public static RepositoriesMetadata fromXContent(XContentParser parser) throws IO
         XContentParser.Token token;
         List<RepositoryMetadata> repository = new ArrayList<>();
         while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (token == XContentParser.Token.START_OBJECT) {
+                // move to next token if parsing the whole object
+                token = parser.nextToken();
+            }
             if (token == XContentParser.Token.FIELD_NAME) {
                 String name = parser.currentName();
                 if (parser.nextToken() != XContentParser.Token.START_OBJECT) {