Make elasticsearch-node tools custom metadata-aware (elastic#48390)

The elasticsearch-node tools allow manipulating the on-disk cluster state. The tool is currently
unaware of plugins and will therefore drop custom metadata from the cluster state once the
state is written out again (as it skips over the custom metadata that it can't read). This commit
preserves unknown customs when editing on-disk metadata through the elasticsearch-node
command-line tools.

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -424,23 +424,23 @@ public synchronized void start() {
 
         if (plugins.isEmpty() == false) {
             logToProcessStdout("Installing " + plugins.size() + " plugins");
-            plugins.forEach(plugin -> runElaticsearchBinScript(
+            plugins.forEach(plugin -> runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", plugin.toString())
             );
         }
 
         if (getVersion().before("6.3.0") && testDistribution == TestDistribution.DEFAULT) {
             LOGGER.info("emulating the {} flavor for {} by installing x-pack", testDistribution, getVersion());
-            runElaticsearchBinScript(
+            runElasticsearchBinScript(
                 "elasticsearch-plugin",
                 "install", "--batch", "x-pack"
             );
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
-            runElaticsearchBinScript("elasticsearch-keystore", "create");
+            runElasticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
                 runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
@@ -452,7 +452,7 @@ public synchronized void start() {
                 if (file.exists() == false) {
                     throw new TestClustersException("supplied keystore file " + file + " does not exist, require for " + this);
                 }
-                runElaticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
+                runElasticsearchBinScript("elasticsearch-keystore", "add-file", entry.getKey(), file.getAbsolutePath());
             }
         }
 
@@ -467,7 +467,7 @@ public synchronized void start() {
         if (credentials.isEmpty() == false) {
             logToProcessStdout("Setting up " + credentials.size() + " users");
 
-            credentials.forEach(paramMap -> runElaticsearchBinScript(
+            credentials.forEach(paramMap -> runElasticsearchBinScript(
                 getVersion().onOrAfter("6.3.0") ? "elasticsearch-users" : "x-pack/users",
                 paramMap.entrySet().stream()
                     .flatMap(entry -> Stream.of(entry.getKey(), entry.getValue()))
@@ -663,7 +663,7 @@ private void runElasticsearchBinScriptWithInput(String input, String tool, Strin
         }
     }
 
-    private void runElaticsearchBinScript(String tool, String... args) {
+    private void runElasticsearchBinScript(String tool, String... args) {
         runElasticsearchBinScriptWithInput("", tool, args);
     }
 

libs/x-content/src/main/java/org/elasticsearch/common/xcontent/XContentBuilder.java

@@ -819,7 +819,7 @@ private void unknownValue(Object value, boolean ensureNoSelfReferences) throws I
         } else if (value instanceof Map) {
             @SuppressWarnings("unchecked")
             final Map<String, ?> valueMap = (Map<String, ?>) value;
-            map(valueMap, ensureNoSelfReferences);
+            map(valueMap, ensureNoSelfReferences, true);
         } else if (value instanceof Iterable) {
             value((Iterable<?>) value, ensureNoSelfReferences);
         } else if (value instanceof Object[]) {
@@ -867,10 +867,15 @@ public XContentBuilder field(String name, Map<String, Object> values) throws IOE
     }
 
     public XContentBuilder map(Map<String, ?> values) throws IOException {
-        return map(values, true);
+        return map(values, true, true);
     }
 
-    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences) throws IOException {
+    /** writes a map without the start object and end object headers */
+    public XContentBuilder mapContents(Map<String, ?> values) throws IOException {
+        return map(values, true, false);
+    }
+
+    private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReferences, boolean writeStartAndEndHeaders) throws IOException {
         if (values == null) {
             return nullValue();
         }
@@ -881,13 +886,17 @@ private XContentBuilder map(Map<String, ?> values, boolean ensureNoSelfReference
             ensureNoSelfReferences(values);
         }
 
-        startObject();
+        if (writeStartAndEndHeaders) {
+            startObject();
+        }
         for (Map.Entry<String, ?> value : values.entrySet()) {
             field(value.getKey());
             // pass ensureNoSelfReferences=false as we already performed the check at a higher level
             unknownValue(value.getValue(), false);
         }
-        endObject();
+        if (writeStartAndEndHeaders) {
+            endObject();
+        }
         return this;
     }
 

server/src/main/java/org/elasticsearch/cli/EnvironmentAwareCommand.java

@@ -88,14 +88,19 @@ protected void execute(Terminal terminal, OptionSet options) throws Exception {
 
     /** Create an {@link Environment} for the command to use. Overrideable for tests. */
     protected Environment createEnv(final Map<String, String> settings) throws UserException {
+        return createEnv(Settings.EMPTY, settings);
+    }
+
+    /** Create an {@link Environment} for the command to use. Overrideable for tests. */
+    protected final Environment createEnv(final Settings baseSettings, final Map<String, String> settings) throws UserException {
         final String esPathConf = System.getProperty("es.path.conf");
         if (esPathConf == null) {
             throw new UserException(ExitCodes.CONFIG, "the system property [es.path.conf] must be set");
         }
-        return InternalSettingsPreparer.prepareEnvironment(Settings.EMPTY, settings,
-                getConfigPath(esPathConf),
-                // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
-                () -> System.getenv("HOSTNAME"));
+        return InternalSettingsPreparer.prepareEnvironment(baseSettings, settings,
+            getConfigPath(esPathConf),
+            // HOSTNAME is set by elasticsearch-env and elasticsearch-env.bat so it is always available
+            () -> System.getenv("HOSTNAME"));
     }
 
     @SuppressForbidden(reason = "need path to construct environment")

server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.cli.EnvironmentAwareCommand;
 import org.elasticsearch.cli.Terminal;
-import org.elasticsearch.cluster.ClusterModule;
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.metadata.MetaData;
 import org.elasticsearch.common.collect.Tuple;
@@ -42,7 +41,6 @@
 
 public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
     private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class);
-    protected final NamedXContentRegistry namedXContentRegistry;
     protected static final String DELIMITER = "------------------------------------------------------------------------\n";
 
     static final String STOP_WARNING_MSG =
@@ -61,7 +59,6 @@ public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand {
 
     public ElasticsearchNodeCommand(String description) {
         super(description);
-        namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables());
     }
 
     protected void processNodePaths(Terminal terminal, OptionSet options, Environment env) throws IOException {
@@ -80,7 +77,7 @@ protected void processNodePaths(Terminal terminal, OptionSet options, Environmen
 
     protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file");
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
 
         if (manifest == null) {
             throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG);
@@ -89,8 +86,8 @@ protected Tuple<Manifest, MetaData> loadMetaData(Terminal terminal, Path[] dataP
             throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG);
         }
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file");
-        final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(),
-                dataPaths);
+        final MetaData metaData = MetaData.FORMAT_PRESERVE_CUSTOMS.loadGeneration(
+            logger, NamedXContentRegistry.EMPTY, manifest.getGlobalGeneration(), dataPaths);
         if (metaData == null) {
             throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]");
         }

server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeMetaData;
 import org.elasticsearch.node.Node;
@@ -84,7 +85,7 @@ protected boolean validateBeforeLock(Terminal terminal, Environment env) {
 
     protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata");
-        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
         if (nodeMetaData == null) {
             throw new ElasticsearchException(NO_NODE_METADATA_FOUND_MSG);
         }

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

@@ -45,7 +45,6 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -1421,8 +1420,6 @@ public void toXContent(XContentBuilder builder, IndexMetaData state) throws IOEx
 
         @Override
         public IndexMetaData fromXContent(XContentParser parser) throws IOException {
-            assert parser.getXContentRegistry() != NamedXContentRegistry.EMPTY
-                    : "loading index metadata requires a working named xcontent registry";
             return Builder.fromXContent(parser);
         }
     };

server/src/main/java/org/elasticsearch/cluster/metadata/MetaData.java

@@ -755,7 +755,7 @@ public static Diff<MetaData> readDiffFrom(StreamInput in) throws IOException {
     }
 
     public static MetaData fromXContent(XContentParser parser) throws IOException {
-        return Builder.fromXContent(parser);
+        return Builder.fromXContent(parser, false);
     }
 
     @Override
@@ -1277,7 +1277,7 @@ public static void toXContent(MetaData metaData, XContentBuilder builder, ToXCon
             builder.endObject();
         }
 
-        public static MetaData fromXContent(XContentParser parser) throws IOException {
+        public static MetaData fromXContent(XContentParser parser, boolean preserveUnknownCustoms) throws IOException {
             Builder builder = new Builder();
 
             // we might get here after the meta-data element, or on a fresh parser
@@ -1327,8 +1327,13 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
                             Custom custom = parser.namedObject(Custom.class, currentFieldName, null);
                             builder.putCustom(custom.getWriteableName(), custom);
                         } catch (NamedObjectNotFoundException ex) {
-                            logger.warn("Skipping unknown custom object with type {}", currentFieldName);
-                            parser.skipChildren();
+                            if (preserveUnknownCustoms) {
+                                logger.warn("Adding unknown custom object with type {}", currentFieldName);
+                                builder.putCustom(currentFieldName, new UnknownGatewayOnlyCustom(parser.mapOrdered()));
+                            } else {
+                                logger.warn("Skipping unknown custom object with type {}", currentFieldName);
+                                parser.skipChildren();
+                            }
                         }
                     }
                 } else if (token.isValue()) {
@@ -1349,6 +1354,45 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
         }
     }
 
+    public static class UnknownGatewayOnlyCustom implements Custom {
+
+        private final Map<String, Object> contents;
+
+        UnknownGatewayOnlyCustom(Map<String, Object> contents) {
+            this.contents = contents;
+        }
+
+        @Override
+        public EnumSet<XContentContext> context() {
+            return EnumSet.of(MetaData.XContentContext.API, MetaData.XContentContext.GATEWAY);
+        }
+
+        @Override
+        public Diff<Custom> diff(Custom previousState) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String getWriteableName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Version getMinimalSupportedVersion() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void writeTo(StreamOutput out) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            return builder.mapContents(contents);
+        }
+    }
+
     private static final ToXContent.Params FORMAT_PARAMS;
     static {
         Map<String, String> params = new HashMap<>(2);
@@ -1360,16 +1404,25 @@ public static MetaData fromXContent(XContentParser parser) throws IOException {
     /**
      * State format for {@link MetaData} to write to and load from disk
      */
-    public static final MetaDataStateFormat<MetaData> FORMAT = new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
+    public static final MetaDataStateFormat<MetaData> FORMAT = createMetaDataStateFormat(false);
 
-        @Override
-        public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
-            Builder.toXContent(state, builder, FORMAT_PARAMS);
-        }
+    /**
+     * Special state format for {@link MetaData} to write to and load from disk, preserving unknown customs
+     */
+    public static final MetaDataStateFormat<MetaData> FORMAT_PRESERVE_CUSTOMS = createMetaDataStateFormat(true);
 
-        @Override
-        public MetaData fromXContent(XContentParser parser) throws IOException {
-            return Builder.fromXContent(parser);
-        }
-    };
+    private static MetaDataStateFormat<MetaData> createMetaDataStateFormat(boolean preserveUnknownCustoms) {
+        return new MetaDataStateFormat<MetaData>(GLOBAL_STATE_FILE_PREFIX) {
+
+            @Override
+            public void toXContent(XContentBuilder builder, MetaData state) throws IOException {
+                Builder.toXContent(state, builder, FORMAT_PARAMS);
+            }
+
+            @Override
+            public MetaData fromXContent(XContentParser parser) throws IOException {
+                return Builder.fromXContent(parser, preserveUnknownCustoms);
+            }
+        };
+    }
 }

server/src/main/java/org/elasticsearch/env/NodeRepurposeCommand.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.metadata.Manifest;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.WriteStateException;
 
@@ -165,7 +166,7 @@ private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) {
             indexPaths[i] = nodePaths[i].resolve(uuid);
         }
         try {
-            IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, indexPaths);
+            IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, indexPaths);
             return metaData.getIndex().getName();
         } catch (Exception e) {
             return "no name for uuid: " + uuid + ": " + e;
@@ -194,7 +195,7 @@ private void rewriteManifest(Terminal terminal, Manifest manifest, Path[] dataPa
 
     private Manifest loadManifest(Terminal terminal, Path[] dataPaths) throws IOException {
         terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest");
-        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths);
+        final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, NamedXContentRegistry.EMPTY, dataPaths);
 
         if (manifest == null) {
             terminal.println(Terminal.Verbosity.SILENT, PRE_V7_MESSAGE);

server/src/main/java/org/elasticsearch/env/OverrideNodeVersionCommand.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.Version;
 import org.elasticsearch.cli.Terminal;
 import org.elasticsearch.cluster.coordination.ElasticsearchNodeCommand;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -74,7 +75,7 @@ public OverrideNodeVersionCommand() {
     protected void processNodePaths(Terminal terminal, Path[] dataPaths, Environment env) throws IOException {
         final Path[] nodePaths = Arrays.stream(toNodePaths(dataPaths)).map(p -> p.path).toArray(Path[]::new);
         final NodeMetaData nodeMetaData
-            = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, namedXContentRegistry, nodePaths);
+            = new NodeMetaData.NodeMetaDataStateFormat(true).loadLatestState(logger, NamedXContentRegistry.EMPTY, nodePaths);
         if (nodeMetaData == null) {
             throw new ElasticsearchException(NO_METADATA_MESSAGE);
         }

server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataTests.java

@@ -365,7 +365,7 @@ public void testUnknownFieldClusterMetaData() throws IOException {
                 .endObject()
             .endObject());
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, metadata)) {
-            MetaData.Builder.fromXContent(parser);
+            MetaData.Builder.fromXContent(parser, randomBoolean());
             fail();
         } catch (IllegalArgumentException e) {
             assertEquals("Unexpected field [random]", e.getMessage());

server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetaDataTests.java

@@ -140,7 +140,7 @@ public void testSimpleJsonFromAndTo() throws IOException {
 
         String metaDataSource = MetaData.Builder.toXContent(metaData);
 
-        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource));
+        MetaData parsedMetaData = MetaData.Builder.fromXContent(createParser(JsonXContent.jsonXContent, metaDataSource), false);
 
         IndexMetaData indexMetaData = parsedMetaData.index("test1");
         assertThat(indexMetaData.primaryTerm(0), equalTo(1L));