Upload all the metadata attributes in parallel

Signed-off-by: Shivansh Arora <hishiv@amazon.com>

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -752,6 +752,10 @@ public String getIndexUUID() {
         return index.getUUID();
     }
 
+    public String getIndexName() {
+        return index.getName();
+    }
+
     /**
      * Test whether the current index UUID is the same as the given one. Returns true if either are _na_
      */

server/src/main/java/org/opensearch/cluster/metadata/Metadata.java

@@ -948,7 +948,7 @@ public Iterator<IndexMetadata> iterator() {
     }
 
     public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata)) {
+        if (!isCoordinationMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         if (!metadata1.hashesOfConsistentSettings.equals(metadata2.hashesOfConsistentSettings)) {
@@ -967,13 +967,29 @@ public static boolean isGlobalStateEquals(Metadata metadata1, Metadata metadata2
      * Compares Metadata entities persisted in Remote Store.
      */
     public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metadata metadata2) {
-        if (!metadata1.persistentSettings.equals(metadata2.persistentSettings)) {
+        if (!isSettingsMetadataEqual(metadata1, metadata2)) {
             return false;
         }
-        if (!metadata1.templates.equals(metadata2.templates)) {
+        if (!isTemplatesMetadataEqual(metadata1, metadata2)) {
             return false;
         }
         // Check if any persistent metadata needs to be saved
+        return isCustomMetadataEqual(metadata1, metadata2);
+    }
+
+    public static boolean isCoordinationMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.coordinationMetadata.equals(metadata2.coordinationMetadata);
+    }
+
+    public static boolean isSettingsMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.persistentSettings.equals(metadata2.persistentSettings);
+    }
+
+    public static boolean isTemplatesMetadataEqual(Metadata metadata1, Metadata metadata2) {
+        return metadata1.templates.equals(metadata2.templates);
+    }
+
+    public static boolean isCustomMetadataEqual(Metadata metadata1, Metadata metadata2) {
         int customCount1 = 0;
         for (Map.Entry<String, Custom> cursor : metadata1.customs.entrySet()) {
             if (cursor.getValue().context().contains(XContentContext.GATEWAY)) {
@@ -987,8 +1003,7 @@ public static boolean isGlobalResourcesMetadataEquals(Metadata metadata1, Metada
                 customCount2++;
             }
         }
-        if (customCount1 != customCount2) return false;
-        return true;
+        return customCount1 == customCount2;
     }
 
     @Override

server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java

@@ -24,6 +24,7 @@
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -299,7 +300,7 @@ public Map<String, UploadedMetadataAttribute> getCustomMetadataMap() {
     }
 
     public boolean hasMetadataAttributesFiles() {
-        return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || uploadedCustomMetadataMap != null;
+        return uploadedCoordinationMetadata != null || uploadedSettingsMetadata != null || uploadedTemplatesMetadata != null || !uploadedCustomMetadataMap.isEmpty();
     }
 
     public ClusterMetadataManifest(
@@ -369,7 +370,7 @@ public ClusterMetadataManifest(
         this.uploadedCoordinationMetadata = uploadedCoordinationMetadata;
         this.uploadedSettingsMetadata = uploadedSettingsMetadata;
         this.uploadedTemplatesMetadata = uploadedTemplatesMetadata;
-        this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap);
+        this.uploadedCustomMetadataMap = Collections.unmodifiableMap(uploadedCustomMetadataMap != null ? uploadedCustomMetadataMap : new HashMap<>());
     }
 
     public ClusterMetadataManifest(StreamInput in) throws IOException {
@@ -534,6 +535,10 @@ public static ClusterMetadataManifest fromXContentV0(XContentParser parser) thro
         return PARSER_V0.parse(parser, null);
     }
 
+    public static ClusterMetadataManifest fromXContentV1(XContentParser parser) throws IOException {
+        return PARSER_V1.parse(parser, null);
+    }
+
     public static ClusterMetadataManifest fromXContent(XContentParser parser) throws IOException {
         return CURRENT_PARSER.parse(parser, null);
     }
@@ -546,6 +551,10 @@ public static ClusterMetadataManifest fromXContent(XContentParser parser) throws
     public static class Builder {
 
         private String globalMetadataFileName;
+        private UploadedMetadataAttribute coordinationMetadata;
+        private UploadedMetadataAttribute settingsMetadata;
+        private UploadedMetadataAttribute templatesMetadata;
+        private Map<String, UploadedMetadataAttribute> customMetadataMap;
         private int codecVersion;
         private List<UploadedIndexMetadata> indices;
         private long clusterTerm;
@@ -573,6 +582,31 @@ public Builder globalMetadataFileName(String globalMetadataFileName) {
             return this;
         }
 
+        public Builder coordinationMetadata(UploadedMetadataAttribute coordinationMetadata) {
+            this.coordinationMetadata = coordinationMetadata;
+            return this;
+        }
+
+        public Builder settingMetadata(UploadedMetadataAttribute settingsMetadata) {
+            this.settingsMetadata = settingsMetadata;
+            return this;
+        }
+
+        public Builder templatesMetadata(UploadedMetadataAttribute templatesMetadata) {
+            this.templatesMetadata = templatesMetadata;
+            return this;
+        }
+
+        public Builder customMetadataMap(Map<String, UploadedMetadataAttribute> customMetadataMap) {
+            this.customMetadataMap = customMetadataMap;
+            return this;
+        }
+
+        public Builder put(String custom, UploadedMetadataAttribute customMetadata) {
+            this.customMetadataMap.put(custom, customMetadata);
+            return this;
+        }
+
         public Builder clusterTerm(long clusterTerm) {
             this.clusterTerm = clusterTerm;
             return this;
@@ -624,6 +658,7 @@ public Builder clusterUUIDCommitted(boolean clusterUUIDCommitted) {
 
         public Builder() {
             indices = new ArrayList<>();
+            customMetadataMap = new HashMap<>();
         }
 
         public Builder(ClusterMetadataManifest manifest) {
@@ -635,6 +670,10 @@ public Builder(ClusterMetadataManifest manifest) {
             this.nodeId = manifest.nodeId;
             this.committed = manifest.committed;
             this.globalMetadataFileName = manifest.globalMetadataFileName;
+            this.coordinationMetadata = manifest.uploadedCoordinationMetadata;
+            this.settingsMetadata = manifest.uploadedSettingsMetadata;
+            this.templatesMetadata = manifest.uploadedTemplatesMetadata;
+            this.customMetadataMap = manifest.uploadedCustomMetadataMap;
             this.codecVersion = manifest.codecVersion;
             this.indices = new ArrayList<>(manifest.indices);
             this.previousClusterUUID = manifest.previousClusterUUID;
@@ -654,18 +693,27 @@ public ClusterMetadataManifest build() {
                 globalMetadataFileName,
                 indices,
                 previousClusterUUID,
-                clusterUUIDCommitted
+                clusterUUIDCommitted,
+                coordinationMetadata,
+                settingsMetadata,
+                templatesMetadata,
+                customMetadataMap
             );
         }
 
     }
 
+    public static interface UploadedMetadata {
+        String getComponent();
+        String getUploadedFilename();
+    }
+
     /**
      * Metadata for uploaded index metadata
      *
      * @opensearch.internal
      */
-    public static class UploadedIndexMetadata implements Writeable, ToXContentFragment {
+    public static class UploadedIndexMetadata implements UploadedMetadata, Writeable, ToXContentFragment {
 
         private static final ParseField INDEX_NAME_FIELD = new ParseField("index_name");
         private static final ParseField INDEX_UUID_FIELD = new ParseField("index_uuid");
@@ -714,6 +762,11 @@ public String getUploadedFilePath() {
             return uploadedFilename;
         }
 
+        @Override
+        public String getComponent() {
+            return getIndexName();
+        }
+
         public String getUploadedFilename() {
             String[] splitPath = uploadedFilename.split("/");
             return splitPath[splitPath.length - 1];
@@ -772,7 +825,7 @@ public static UploadedIndexMetadata fromXContent(XContentParser parser) throws I
         }
     }
 
-    public static class UploadedMetadataAttribute implements Writeable, ToXContentFragment {
+    public static class UploadedMetadataAttribute implements UploadedMetadata, Writeable, ToXContentFragment {
         private static final ParseField UPLOADED_FILENAME_FIELD = new ParseField("uploaded_filename");
 
         private static final ObjectParser.NamedObjectParser<UploadedMetadataAttribute, Void> PARSER;
@@ -807,6 +860,11 @@ public String getAttributeName() {
             return attributeName;
         }
 
+        @Override
+        public String getComponent() {
+            return getAttributeName();
+        }
+
         public String getUploadedFilename() {
             return uploadedFilename;
         }